mirror of
https://github.com/anomalyco/opencode.git
synced 2026-11-17 05:26:47 +03:00
fix(core): keep grep results when a match is not valid UTF-8
This commit is contained in:
@@ -20,16 +20,19 @@ const ERROR_BYTES = 8 * 1024
|
||||
const MAX_SUBMATCHES = 100
|
||||
const MAX_LINE_CHARS = 2_000
|
||||
|
||||
// rg reports data that is not valid UTF-8 as base64 `bytes` instead of `text`.
|
||||
const RawData = Schema.Union([Schema.Struct({ text: Schema.String }), Schema.Struct({ bytes: Schema.String })])
|
||||
|
||||
const RawMatch = Schema.Struct({
|
||||
type: Schema.Literal("match"),
|
||||
data: Schema.Struct({
|
||||
path: Schema.Struct({ text: Schema.String }),
|
||||
lines: Schema.Struct({ text: Schema.String }),
|
||||
path: RawData,
|
||||
lines: RawData,
|
||||
line_number: PositiveInt,
|
||||
absolute_offset: NonNegativeInt,
|
||||
submatches: Schema.Array(
|
||||
Schema.Struct({
|
||||
match: Schema.Struct({ text: Schema.String }),
|
||||
match: RawData,
|
||||
start: NonNegativeInt,
|
||||
end: NonNegativeInt,
|
||||
}),
|
||||
@@ -38,7 +41,8 @@ const RawMatch = Schema.Struct({
|
||||
})
|
||||
const decodeJsonRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown))
|
||||
|
||||
type RawMatchData = (typeof RawMatch.Type)["data"]
|
||||
const rawText = (data: typeof RawData.Type) =>
|
||||
"text" in data ? data.text : Buffer.from(data.bytes, "base64").toString("utf8")
|
||||
|
||||
export class Error extends Schema.TaggedError<Error>()("Ripgrep.Error", {
|
||||
message: Schema.String,
|
||||
@@ -218,7 +222,7 @@ const layer = Layer.effect(
|
||||
onItem: input.onEntry,
|
||||
}).pipe(Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause)))),
|
||||
grep: (input) =>
|
||||
run<RawMatchData>({
|
||||
run({
|
||||
...input,
|
||||
args: [
|
||||
"--no-config",
|
||||
@@ -240,17 +244,23 @@ const layer = Layer.effect(
|
||||
if (!json || typeof json !== "object" || !("type" in json) || json.type !== "match")
|
||||
return Effect.undefined
|
||||
return Schema.decodeUnknownEffect(RawMatch)(json).pipe(
|
||||
Effect.map((match) => ({
|
||||
...match.data,
|
||||
lines: {
|
||||
text:
|
||||
match.data.lines.text.length > MAX_LINE_CHARS
|
||||
? Buffer.from(match.data.lines.text.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
|
||||
: match.data.lines.text,
|
||||
},
|
||||
path: { text: normalizePath(match.data.path.text) },
|
||||
submatches: match.data.submatches.slice(0, MAX_SUBMATCHES),
|
||||
})),
|
||||
Effect.map((match) => {
|
||||
const lines = rawText(match.data.lines)
|
||||
return {
|
||||
...match.data,
|
||||
lines: {
|
||||
text:
|
||||
lines.length > MAX_LINE_CHARS
|
||||
? Buffer.from(lines.slice(0, MAX_LINE_CHARS), "utf8").toString() + "..."
|
||||
: lines,
|
||||
},
|
||||
path: { text: normalizePath(rawText(match.data.path)) },
|
||||
submatches: match.data.submatches.slice(0, MAX_SUBMATCHES).map((submatch) => ({
|
||||
...submatch,
|
||||
match: { text: rawText(submatch.match) },
|
||||
})),
|
||||
}
|
||||
}),
|
||||
Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)),
|
||||
)
|
||||
}),
|
||||
|
||||
@@ -78,6 +78,24 @@ describe("Ripgrep", () => {
|
||||
),
|
||||
)
|
||||
|
||||
it.live("greps lines that are not valid UTF-8", () =>
|
||||
withTempDir((tmp) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.promise(() => fs.writeFile(path.join(tmp.path, "utf8.txt"), "needle\n"))
|
||||
yield* Effect.promise(() =>
|
||||
fs.writeFile(path.join(tmp.path, "latin1.txt"), Buffer.from([...Buffer.from("needle caf"), 0xe9, 0x0a])),
|
||||
)
|
||||
|
||||
const ripgrep = yield* Ripgrep.Service
|
||||
const result = yield* ripgrep.grep({ cwd: tmp.path, pattern: "needle", limit: 10 })
|
||||
expect(result.map((match) => [match.entry.path, match.text, match.submatches[0]?.text]).sort()).toEqual([
|
||||
[RelativePath.make("latin1.txt"), "needle caf\uFFFD\n", "needle"],
|
||||
[RelativePath.make("utf8.txt"), "needle\n", "needle"],
|
||||
])
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
it.live("keeps ignored files out of catch-all find results", () =>
|
||||
withTempDir((tmp) =>
|
||||
Effect.gen(function* () {
|
||||
|
||||
Reference in New Issue
Block a user