fix(core): keep grep results when a match is not valid UTF-8

This commit is contained in:
Daniel Smolsky
2026-11-05 22:42:35 -05:00
Unverified
parent 2658ec02c8
commit 0638a57ff1
2 changed files with 44 additions and 16 deletions
+26 -16
View File
@@ -20,16 +20,19 @@ const ERROR_BYTES = 8 * 1024
const MAX_SUBMATCHES = 100
const MAX_LINE_CHARS = 2_000
// rg reports data that is not valid UTF-8 as base64 `bytes` instead of `text`.
const RawData = Schema.Union([Schema.Struct({ text: Schema.String }), Schema.Struct({ bytes: Schema.String })])
const RawMatch = Schema.Struct({
type: Schema.Literal("match"),
data: Schema.Struct({
path: Schema.Struct({ text: Schema.String }),
lines: Schema.Struct({ text: Schema.String }),
path: RawData,
lines: RawData,
line_number: PositiveInt,
absolute_offset: NonNegativeInt,
submatches: Schema.Array(
Schema.Struct({
match: Schema.Struct({ text: Schema.String }),
match: RawData,
start: NonNegativeInt,
end: NonNegativeInt,
}),
@@ -38,7 +41,8 @@ const RawMatch = Schema.Struct({
})
const decodeJsonRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))
type RawMatchData = (typeof RawMatch.Type)["data"]
const rawText = (data: typeof RawData.Type) =>
"text" in data ? data.text : Buffer.from(data.bytes, "base64").toString("utf8")
export class Error extends Schema.TaggedError<Error>()("Ripgrep.Error", {
message: Schema.String,
@@ -218,7 +222,7 @@ const layer = Layer.effect(
onItem: input.onEntry,
}).pipe(Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause)))),
grep: (input) =>
run<RawMatchData>({
run({
...input,
args: [
"--no-config",
@@ -240,17 +244,23 @@ const layer = Layer.effect(
if (!json || typeof json !== "object" || !("type" in json) || json.type !== "match")
return Effect.undefined
return Schema.decodeUnknownEffect(RawMatch)(json).pipe(
Effect.map((match) => ({
...match.data,
lines: {
text:
match.data.lines.text.length > MAX_LINE_CHARS
? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
: match.data.lines.text,
},
path: { text: normalizePath(match.data.path.text) },
submatches: match.data.submatches.slice(0, MAX_SUBMATCHES),
})),
Effect.map((match) => {
const lines = rawText(match.data.lines)
return {
...match.data,
lines: {
text:
lines.length > MAX_LINE_CHARS
? Buffer.from(lines.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
: lines,
},
path: { text: normalizePath(rawText(match.data.path)) },
submatches: match.data.submatches.slice(0, MAX_SUBMATCHES).map((submatch) => ({
...submatch,
match: { text: rawText(submatch.match) },
})),
}
}),
Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)),
)
}),
+18
View File
@@ -78,6 +78,24 @@ describe("Ripgrep", () => {
),
)
it.live("greps lines that are not valid UTF-8", () =>
withTempDir((tmp) =>
Effect.gen(function* () {
yield* Effect.promise(() => fs.writeFile(path.join(tmp.path, "utf8.txt"), "needle\n"))
yield* Effect.promise(() =>
fs.writeFile(path.join(tmp.path, "latin1.txt"), Buffer.from([...Buffer.from("needle caf"), 0xe9, 0x0a])),
)
const ripgrep = yield* Ripgrep.Service
const result = yield* ripgrep.grep({ cwd: tmp.path, pattern: "needle", limit: 10 })
expect(result.map((match) => [match.entry.path, match.text, match.submatches[0]?.text]).sort()).toEqual([
[RelativePath.make("latin1.txt"), "needle caf\uFFFD\n", "needle"],
[RelativePath.make("utf8.txt"), "needle\n", "needle"],
])
}),
),
)
it.live("keeps ignored files out of catch-all find results", () =>
withTempDir((tmp) =>
Effect.gen(function* () {