diff --git a/packages/core/src/ripgrep.ts b/packages/core/src/ripgrep.ts index 5e49d46223c..a5a142823d5 100644 --- a/packages/core/src/ripgrep.ts +++ b/packages/core/src/ripgrep.ts @@ -20,16 +20,19 @@ const ERROR_BYTES = 8 * 1024 const MAX_SUBMATCHES = 100 const MAX_LINE_CHARS = 2_000 +// rg reports data that is not valid UTF-8 as base64 `bytes` instead of `text`. +const RawData = Schema.Union([Schema.Struct({ text: Schema.String }), Schema.Struct({ bytes: Schema.String })]) + const RawMatch = Schema.Struct({ type: Schema.Literal("match"), data: Schema.Struct({ - path: Schema.Struct({ text: Schema.String }), - lines: Schema.Struct({ text: Schema.String }), + path: RawData, + lines: RawData, line_number: PositiveInt, absolute_offset: NonNegativeInt, submatches: Schema.Array( Schema.Struct({ - match: Schema.Struct({ text: Schema.String }), + match: RawData, start: NonNegativeInt, end: NonNegativeInt, }), @@ -38,7 +41,8 @@ const RawMatch = Schema.Struct({ }) const decodeJsonRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown)) -type RawMatchData = (typeof RawMatch.Type)["data"] +const rawText = (data: typeof RawData.Type) => + "text" in data ? data.text : Buffer.from(data.bytes, "base64").toString("utf8") export class Error extends Schema.TaggedError()("Ripgrep.Error", { message: Schema.String, @@ -218,7 +222,7 @@ const layer = Layer.effect( onItem: input.onEntry, }).pipe(Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause)))), grep: (input) => - run({ + run({ ...input, args: [ "--no-config", @@ -240,17 +244,23 @@ const layer = Layer.effect( if (!json || typeof json !== "object" || !("type" in json) || json.type !== "match") return Effect.undefined return Schema.decodeUnknownEffect(RawMatch)(json).pipe( - Effect.map((match) => ({ - ...match.data, - lines: { - text: - match.data.lines.text.length > MAX_LINE_CHARS - ? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..." - : match.data.lines.text, - }, - path: { text: normalizePath(match.data.path.text) }, - submatches: match.data.submatches.slice(0, MAX_SUBMATCHES), - })), + Effect.map((match) => { + const lines = rawText(match.data.lines) + return { + ...match.data, + lines: { + text: + lines.length > MAX_LINE_CHARS + ? Buffer.from(lines.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..." + : lines, + }, + path: { text: normalizePath(rawText(match.data.path)) }, + submatches: match.data.submatches.slice(0, MAX_SUBMATCHES).map((submatch) => ({ + ...submatch, + match: { text: rawText(submatch.match) }, + })), + } + }), Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)), ) }), diff --git a/packages/core/test/ripgrep.test.ts b/packages/core/test/ripgrep.test.ts index b1c7fc0e465..8540483f3f5 100644 --- a/packages/core/test/ripgrep.test.ts +++ b/packages/core/test/ripgrep.test.ts @@ -78,6 +78,24 @@ describe("Ripgrep", () => { ), ) + it.live("greps lines that are not valid UTF-8", () => + withTempDir((tmp) => + Effect.gen(function* () { + yield* Effect.promise(() => fs.writeFile(path.join(tmp.path, "utf8.txt"), "needle\n")) + yield* Effect.promise(() => + fs.writeFile(path.join(tmp.path, "latin1.txt"), Buffer.from([...Buffer.from("needle caf"), 0xe9, 0x0a])), + ) + + const ripgrep = yield* Ripgrep.Service + const result = yield* ripgrep.grep({ cwd: tmp.path, pattern: "needle", limit: 10 }) + expect(result.map((match) => [match.entry.path, match.text, match.submatches[0]?.text]).sort()).toEqual([ + [RelativePath.make("latin1.txt"), "needle caf\uFFFD\n", "needle"], + [RelativePath.make("utf8.txt"), "needle\n", "needle"], + ]) + }), + ), + ) + it.live("keeps ignored files out of catch-all find results", () => withTempDir((tmp) => Effect.gen(function* () {