diff --git a/packages/ai/src/protocols/utils/partial-json-options.ts b/packages/ai/src/protocols/utils/partial-json-options.ts new file mode 100644 index 00000000000..2021d54bf3d --- /dev/null +++ b/packages/ai/src/protocols/utils/partial-json-options.ts @@ -0,0 +1,64 @@ +/* + * Adapted from partial-json by the Promplate Dev Team: + * https://github.com/promplate/partial-json-parser-js/blob/main/src/options.ts + * Licensed under the MIT License; see partial-json.ts for the complete notice. + */ + +/** + * allow partial strings like `"hello \u12` to be parsed as `"hello ` + */ +export const STR = 0b000000001 + +/** + * allow partial numbers like `123.` to be parsed as `123` + */ +export const NUM = 0b000000010 + +/** + * allow partial arrays like `[1, 2,` to be parsed as `[1, 2]` + */ +export const ARR = 0b000000100 + +/** + * allow partial objects like `{"a": 1, "b":` to be parsed as `{"a": 1}` + */ +export const OBJ = 0b000001000 + +/** + * allow `nu` to be parsed as `null` + */ +export const NULL = 0b000010000 + +/** + * allow `tr` to be parsed as `true`, and `fa` to be parsed as `false` + */ +export const BOOL = 0b000100000 + +/** + * allow `Na` to be parsed as `NaN` + */ +export const NAN = 0b001000000 + +/** + * allow `Inf` to be parsed as `Infinity` + */ +export const INFINITY = 0b010000000 + +/** + * allow `-Inf` to be parsed as `-Infinity` + */ +export const _INFINITY = 0b100000000 + +export const INF = INFINITY | _INFINITY +export const SPECIAL = NULL | BOOL | INF | NAN +export const ATOM = STR | NUM | SPECIAL +export const COLLECTION = ARR | OBJ +export const ALL = ATOM | COLLECTION + +/** + * Control what types you allow to be partially parsed. + * The default is to allow all types to be partially parsed, which in most cases is the best option. + */ +export const Allow = { STR, NUM, ARR, OBJ, NULL, BOOL, NAN, INFINITY, _INFINITY, INF, SPECIAL, ATOM, COLLECTION, ALL } + +export default Allow diff --git a/packages/ai/src/protocols/utils/partial-json.ts b/packages/ai/src/protocols/utils/partial-json.ts new file mode 100644 index 00000000000..3ae142ffd7a --- /dev/null +++ b/packages/ai/src/protocols/utils/partial-json.ts @@ -0,0 +1,223 @@ +/* + * Adapted from partial-json by the Promplate Dev Team: + * https://github.com/promplate/partial-json-parser-js + * + * MIT License + * + * Copyright (c) 2023 Promplate Dev Team + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import { Schema } from "effect" +import { Allow } from "./partial-json-options.js" +export * from "./partial-json-options.js" + +export class PartialJSON extends Error {} +export class MalformedJSON extends Error {} + +const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)) + +/** Parse complete or incomplete JSON, restricted by the supplied partial-value flags. */ +export function parseJSON(jsonString: string, allowPartial = Allow.ALL): unknown { + if (typeof jsonString !== "string") throw new TypeError(`expecting str, got ${typeof jsonString}`) + const input = jsonString.trim() + if (!input) throw new Error(`${jsonString} is empty`) + try { + return decodeJson(input) + } catch {} + return _parseJSON(input, allowPartial) +} + +const _parseJSON = (jsonString: string, allow: number) => { + const length = jsonString.length + let index = 0 + + const markPartialJSON = (message: string): never => { + throw new PartialJSON(`${message} at position ${index}`) + } + + const throwMalformedError = (message: string): never => { + throw new MalformedJSON(`${message} at position ${index}`) + } + + const parseAny = (): unknown => { + skipBlank() + if (index >= length) markPartialJSON("Unexpected end of input") + if (jsonString[index] === '"') return parseStr() + if (jsonString[index] === "{") return parseObj() + if (jsonString[index] === "[") return parseArr() + if ( + jsonString.substring(index, index + 4) === "null" || + (Allow.NULL & allow && length - index < 4 && "null".startsWith(jsonString.substring(index))) + ) { + index += 4 + return null + } + if ( + jsonString.substring(index, index + 4) === "true" || + (Allow.BOOL & allow && length - index < 4 && "true".startsWith(jsonString.substring(index))) + ) { + index += 4 + return true + } + if ( + jsonString.substring(index, index + 5) === "false" || + (Allow.BOOL & allow && length - index < 5 && "false".startsWith(jsonString.substring(index))) + ) { + index += 5 + return false + } + if ( + jsonString.substring(index, index + 8) === "Infinity" || + (Allow.INFINITY & allow && length - index < 8 && "Infinity".startsWith(jsonString.substring(index))) + ) { + index += 8 + return Infinity + } + if ( + jsonString.substring(index, index + 9) === "-Infinity" || + (Allow._INFINITY & allow && + 1 < length - index && + length - index < 9 && + "-Infinity".startsWith(jsonString.substring(index))) + ) { + index += 9 + return -Infinity + } + if ( + jsonString.substring(index, index + 3) === "NaN" || + (Allow.NAN & allow && length - index < 3 && "NaN".startsWith(jsonString.substring(index))) + ) { + index += 3 + return NaN + } + return parseNum() + } + + const parseStr = (): string => { + const start = index + let escape = false + index++ + while (index < length && (jsonString[index] !== '"' || (escape && jsonString[index - 1] === "\\"))) { + escape = jsonString[index] === "\\" ? !escape : false + index++ + } + if (jsonString.charAt(index) === '"') { + try { + return decodeJson(jsonString.substring(start, ++index - Number(escape))) as string + } catch (error) { + throwMalformedError(String(error)) + } + } + if (Allow.STR & allow) { + try { + return decodeJson(`${jsonString.substring(start, index - Number(escape))}"`) as string + } catch { + return decodeJson(`${jsonString.substring(start, jsonString.lastIndexOf("\\"))}"`) as string + } + } + return markPartialJSON("Unterminated string literal") + } + + const parseObj = (): Record => { + index++ + skipBlank() + const object: Record = {} + try { + while (jsonString[index] !== "}") { + skipBlank() + if (index >= length && Allow.OBJ & allow) return object + const key = parseStr() + skipBlank() + index++ + try { + object[key] = parseAny() + } catch (error) { + if (Allow.OBJ & allow) return object + throw error + } + skipBlank() + if (jsonString[index] === ",") index++ + } + } catch { + if (Allow.OBJ & allow) return object + return markPartialJSON("Expected '}' at end of object") + } + index++ + return object + } + + const parseArr = (): unknown[] => { + index++ + const array: unknown[] = [] + try { + while (jsonString[index] !== "]") { + array.push(parseAny()) + skipBlank() + if (jsonString[index] === ",") index++ + } + } catch { + if (Allow.ARR & allow) return array + return markPartialJSON("Expected ']' at end of array") + } + index++ + return array + } + + const parseNum = (): unknown => { + if (index === 0) { + if (jsonString === "-") throwMalformedError("Not sure what '-' is") + try { + return decodeJson(jsonString) + } catch (error) { + if (Allow.NUM & allow) { + try { + return decodeJson(jsonString.substring(0, jsonString.lastIndexOf("e"))) + } catch {} + } + throwMalformedError(String(error)) + } + } + + const start = index + if (jsonString[index] === "-") index++ + while (jsonString[index] && !",]}".includes(jsonString[index])) index++ + if (index === length && !(Allow.NUM & allow)) markPartialJSON("Unterminated number literal") + + try { + return decodeJson(jsonString.substring(start, index)) + } catch (error) { + if (jsonString.substring(start, index) === "-") markPartialJSON("Not sure what '-' is") + try { + return decodeJson(jsonString.substring(start, jsonString.lastIndexOf("e"))) + } catch { + throwMalformedError(String(error)) + } + } + } + + const skipBlank = () => { + while (index < length && " \n\r\t".includes(jsonString[index])) index++ + } + + return parseAny() +} + +export const parse = parseJSON diff --git a/packages/ai/test/partial-json.test.ts b/packages/ai/test/partial-json.test.ts new file mode 100644 index 00000000000..7f68686ab92 --- /dev/null +++ b/packages/ai/test/partial-json.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test" +import { Allow, MalformedJSON, PartialJSON, parse } from "../src/protocols/utils/partial-json.js" + +describe("partial JSON", () => { + test("parses complete JSON", () => { + expect(parse('{"key":"value","items":[1,true,null]}')).toEqual({ + key: "value", + items: [1, true, null], + }) + + const object = parse('{"__proto__":{"safe":true}}') as Record + expect(Object.hasOwn(object, "__proto__")).toBe(true) + }) + + test("parses partial strings", () => { + expect(parse('"hello')).toBe("hello") + expect(parse('"hello \\u12')).toBe("hello ") + expect(() => parse('"hello', ~Allow.STR)).toThrow(PartialJSON) + }) + + test("controls partial collection values independently", () => { + expect(parse('["', Allow.ARR)).toEqual([]) + expect(parse('["', Allow.ARR | Allow.STR)).toEqual([""]) + expect(parse('{"key":"', Allow.OBJ)).toEqual({}) + expect(parse('{"key":"', Allow.OBJ | Allow.STR)).toEqual({ key: "" }) + }) + + test("parses partial literals and numbers", () => { + expect(parse("nu", Allow.NULL)).toBeNull() + expect(parse("tr", Allow.BOOL)).toBe(true) + expect(parse("fa", Allow.BOOL)).toBe(false) + expect(parse("1e", Allow.NUM)).toBe(1) + }) + + test("distinguishes disallowed partial values from malformed values", () => { + expect(() => parse("[", Allow.STR)).toThrow(PartialJSON) + expect(() => parse("n", ~Allow.NULL)).toThrow(MalformedJSON) + }) + + test("rejects empty input", () => { + expect(() => parse(" ")).toThrow("is empty") + }) +})