diff --git a/docs/cli.md b/docs/cli.md index 245251a28..7f35eb40d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -166,9 +166,16 @@ rea import-reference-source /absolute/path/to/source ``` The import records hashes and metadata for the supplied files, separately -from observations of the current app. File names do not automatically exclude -files. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore -patterns when you want to exclude selected paths. +from observations of the current app. It honors the source tree's `.gitignore` +and applies default exclusions for common generated and dependency paths such +as `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record the +matched pattern and whether it came from project, default, caller, or sensitive +path policy. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore +patterns when you want to mark selected paths as sensitive; those patterns are +reported separately as configured-secret exclusions. + +Re-import saved graphs whose pattern-based exclusions lack `pattern` before +using them in source-to-bundle comparisons. JavaScript and TypeScript import parsing requires valid UTF-8. Malformed source bytes retain their original hashes and sizes with a decoding diagnostic; REA diff --git a/src/application/ReferenceSourceImport.ts b/src/application/ReferenceSourceImport.ts index 69cdeef9c..47b498397 100644 --- a/src/application/ReferenceSourceImport.ts +++ b/src/application/ReferenceSourceImport.ts @@ -15,7 +15,10 @@ import { type ReferenceSourceImportError, type ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; -import { prepareReferenceSourceImport } from "./ReferenceSourceImportPolicy.js"; +import { + prepareReferenceSourceImport, + type PreparedReferenceSourceImport, +} from "./ReferenceSourceImportPolicy.js"; const failure = ( code: ReferenceSourceImportError["code"], @@ -115,9 +118,47 @@ const sortExclusions = ( [...exclusions].sort((left, right) => { const byPath = compareUnicodeCodePoints(left.path, right.path); if (byPath !== 0) return byPath; - return compareUnicodeCodePoints(left.reason, right.reason); + const byReason = compareUnicodeCodePoints(left.reason, right.reason); + if (byReason !== 0) return byReason; + return compareUnicodeCodePoints( + "pattern" in left ? left.pattern : "", + "pattern" in right ? right.pattern : "", + ); }); +const createShouldExclude = + ( + exclusions: HistoricalSourceGraphInput["exclusions"], + secrets: PreparedReferenceSourceImport["secrets"], + ignored: PreparedReferenceSourceImport["ignored"], + ): ((path: string) => boolean) => + (path) => { + const secretMatch = secrets.test(path); + if (secretMatch.ignored) { + if (!secretMatch.rule) + throw new Error(`Ignored secret path has no matching rule: ${path}`); + exclusions.push({ + path, + reason: "configured-secret", + pattern: secretMatch.rule.pattern, + }); + return true; + } + const match = ignored.test(path); + if (!match.ignored) return false; + if (!match.rule) + throw new Error(`Ignored path has no matching rule: ${path}`); + const reason = match.rule.mark; + if ( + reason !== "project-ignored" && + reason !== "default-ignored" && + reason !== "caller-excluded" + ) + throw new Error(`Ignored path has unknown rule origin: ${path}`); + exclusions.push({ path, reason, pattern: match.rule.pattern }); + return true; + }; + /** * Import a reference source directory into a committed historical source graph. * @@ -135,17 +176,7 @@ export const importReferenceSource = async ( if (isAborted(options.signal)) return err(cancelled()); const exclusions: HistoricalSourceGraphInput["exclusions"] = []; - const shouldExclude = (path: string): boolean => { - if (secrets.ignores(path)) { - exclusions.push({ path, reason: "configured-secret" }); - return true; - } - if (ignored.ignores(path)) { - exclusions.push({ path, reason: "caller-excluded" }); - return true; - } - return false; - }; + const shouldExclude = createShouldExclude(exclusions, secrets, ignored); const [readResult, vcs] = await Promise.all([ readReferenceSource(root, { diff --git a/src/application/ReferenceSourceImportPolicy.ts b/src/application/ReferenceSourceImportPolicy.ts index 55a992d53..b67773f7a 100644 --- a/src/application/ReferenceSourceImportPolicy.ts +++ b/src/application/ReferenceSourceImportPolicy.ts @@ -30,6 +30,16 @@ const failure = ( const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true; +const addMarkedPatterns = ( + matcher: ReturnType, + patterns: readonly string[], + mark: "project-ignored" | "default-ignored" | "caller-excluded", +): void => { + for (const pattern of patterns) + for (const line of pattern.split(/\r?\n/u)) + matcher.add({ pattern: line, mark }); +}; + const resolveRoot = async ( requestedRoot: string, signal?: AbortSignal, @@ -87,11 +97,17 @@ const buildIgnored = async ( present = false; } signal?.throwIfAborted(); - if (present) ignored.add(await readRegularFileText(policyPath, { signal })); - ignored.add([...DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS]); + if (present) { + const text = await readRegularFileText(policyPath, { signal }); + addMarkedPatterns(ignored, [text], "project-ignored"); + } + addMarkedPatterns( + ignored, + DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS, + "default-ignored", + ); for (const path of excludePaths) { - ignored.add(path); - ignored.add(`${path}/`); + addMarkedPatterns(ignored, [path, `${path}/`], "caller-excluded"); } return ignored; }; diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index ad4f18bdd..1286cdde6 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -175,7 +175,11 @@ describe("historical source graph", () => { }, (input: HistoricalSourceGraphInput) => { input.exclusions = [ - { path: "private.env", reason: "configured-secret" }, + { + path: "private.env", + reason: "configured-secret", + pattern: ".env", + }, ]; input.inventory_state = "partial"; }, @@ -188,6 +192,20 @@ describe("historical source graph", () => { } }); + it("commits the matched pattern for each exclusion", () => { + const graphWith = (pattern: string) => + createHistoricalSourceGraph({ + ...graphInput(), + inventory_state: "partial", + exclusions: [ + { path: "private.env", reason: "configured-secret", pattern }, + ], + }); + expect(graphWith(".env").root_sha256).not.toBe( + graphWith("*.env").root_sha256, + ); + }); + it("retains actual external targets and rejects placeholder targets", () => { const input = { ...graphInput(), inventory_state: "partial" as const }; input.entries[2] = symlinkEntry({ @@ -300,7 +318,13 @@ describe("partial historical source graphs", () => { { entries: partial.entries }, { parse_failures: partial.parse_failures }, { - exclusions: [{ path: "secret", reason: "configured-secret" as const }], + exclusions: [ + { + path: "secret", + reason: "configured-secret" as const, + pattern: "secret", + }, + ], }, { limitations: ["Incomplete"] }, { diff --git a/src/domain/referenceSourceGraph.ts b/src/domain/referenceSourceGraph.ts index 4c3a06fa7..197e473bf 100644 --- a/src/domain/referenceSourceGraph.ts +++ b/src/domain/referenceSourceGraph.ts @@ -127,15 +127,22 @@ const parseFailureSchema = z.strictObject({ reason: boundedTextSchema, }); -const exclusionSchema = z.strictObject({ - path: relativePathSchema, - reason: z.enum([ - "configured-secret", - "symlink-escape", - "unreadable", - "caller-excluded", - ]), -}); +const exclusionSchema = z.discriminatedUnion("reason", [ + z.strictObject({ + path: relativePathSchema, + reason: z.enum([ + "configured-secret", + "project-ignored", + "default-ignored", + "caller-excluded", + ]), + pattern: boundedTextSchema, + }), + z.strictObject({ + path: relativePathSchema, + reason: z.enum(["symlink-escape", "unreadable"]), + }), +]); const vcsSchema = z .discriminatedUnion("kind", [ @@ -405,7 +412,10 @@ function checkGraphInvariants( context, ); checkSortedUnique( - graph.exclusions.map(({ path, reason }) => `${path}\u0000${reason}`), + graph.exclusions.map( + (exclusion) => + `${exclusion.path}\u0000${exclusion.reason}\u0000${"pattern" in exclusion ? exclusion.pattern : ""}`, + ), "exclusions", context, ); diff --git a/tests/boundary/cli/referenceSourceExclusions.test.ts b/tests/boundary/cli/referenceSourceExclusions.test.ts new file mode 100644 index 000000000..cbc063285 --- /dev/null +++ b/tests/boundary/cli/referenceSourceExclusions.test.ts @@ -0,0 +1,66 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +describe.skipIf(process.platform === "win32")( + "compiled reference-source exclusion provenance", + () => { + cliTest( + "reports project, default, and explicit secret exclusions", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-reference-exclusions-"); + const logs = join(root, "logs"); + await mkdir(logs); + await Promise.all([ + writeFile(join(root, ".gitignore"), "project.tmp\n"), + writeFile(join(root, "project.tmp"), "project policy\n"), + writeFile(join(logs, "application.log"), "default policy\n"), + writeFile(join(root, "private.txt"), "sensitive local file\n"), + writeFile(join(root, "source.ts"), "export const value = 1;\n"), + ]); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + cwd: root, + environment: { + HOME: root, + USERPROFILE: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + REA_REFERENCE_SECRET_PATTERNS_JSON: '["private.txt"]', + }, + }); + + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.exclusions).toEqual( + expect.arrayContaining([ + { + path: "project.tmp", + reason: "project-ignored", + pattern: "project.tmp", + }, + { + path: "logs/application.log", + reason: "default-ignored", + pattern: "*.log", + }, + { + path: "private.txt", + reason: "configured-secret", + pattern: "private.txt", + }, + ]), + ); + expect(graph.entries).toContainEqual( + expect.objectContaining({ path: "source.ts", kind: "file" }), + ); + }, + ); + }, +); diff --git a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts index fd9e07149..a17525e9e 100644 --- a/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts +++ b/tests/boundary/filesystem/referenceSourceFailureDetails.test.ts @@ -116,6 +116,7 @@ describe("reference import native entry failure details", () => { expect(imported.value.exclusions).toContainEqual({ path: "secret-events.pipe", reason: "configured-secret", + pattern: "*secret*", }); }, ); diff --git a/tests/boundary/filesystem/referenceSourceImport.test.ts b/tests/boundary/filesystem/referenceSourceImport.test.ts index 846bbba4a..287834f00 100644 --- a/tests/boundary/filesystem/referenceSourceImport.test.ts +++ b/tests/boundary/filesystem/referenceSourceImport.test.ts @@ -335,6 +335,7 @@ describe("reference source import behavior", () => { expect(left.value.exclusions).toContainEqual({ path: ".env", reason: "configured-secret", + pattern: ".env", }); expect(JSON.stringify(left.value)).not.toContain("SECRET_SENTINEL"); expect(JSON.stringify(left.value)).not.toContain(leftRoot); @@ -435,3 +436,80 @@ describe("reference source rooted module specifiers", () => { }); }); }); + +describe.skipIf(process.platform === "win32")( + "reference source exclusion provenance", + () => { + it("preserves ordered ignore rules and reports each winning rule origin", async () => { + const root = await createTestTempDirectory( + "rea-reference-ignore-origin-", + ); + await mkdir(join(root, "dist"), { recursive: true }); + await mkdir(join(root, "project-parent"), { recursive: true }); + await mkdir(join(root, "caller-parent"), { recursive: true }); + await Promise.all([ + writeFile( + join(root, ".gitignore"), + "project.tmp\n*.log\n!keep.log\nproject-parent/\n!project-parent/keep.ts\n!default.log\n", + ), + writeFile(join(root, "project.tmp"), "project rule\n"), + writeFile(join(root, "project.log"), "project rule\n"), + writeFile(join(root, "keep.log"), "later default rule wins\n"), + writeFile(join(root, "default.log"), "default rule\n"), + writeFile(join(root, "caller-keep.log"), "caller negation\n"), + writeFile(join(root, "dist", "output.js"), "generated output\n"), + writeFile(join(root, "project-parent", "keep.ts"), "parent ignored\n"), + writeFile(join(root, "caller-parent", "keep.ts"), "caller parent\n"), + writeFile(join(root, "selected.ts"), "selected source\n"), + ]); + + const imported = await importReferenceSource({ + root, + caller: "reference-import-test", + policy: { secretPatterns: ["project.log"] }, + excludePaths: ["caller-parent/", "!selected.ts", "!caller-keep.log"], + }); + if (!imported.ok) throw imported.error; + + expect(imported.value.exclusions).toEqual( + expect.arrayContaining([ + { + path: "project.tmp", + reason: "project-ignored", + pattern: "project.tmp", + }, + { + path: "project.log", + reason: "configured-secret", + pattern: "project.log", + }, + { path: "keep.log", reason: "default-ignored", pattern: "*.log" }, + { + path: "dist/output.js", + reason: "default-ignored", + pattern: "dist/", + }, + { + path: "project-parent/keep.ts", + reason: "project-ignored", + pattern: "project-parent/", + }, + { + path: "caller-parent/keep.ts", + reason: "caller-excluded", + pattern: "caller-parent/", + }, + ]), + ); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ path: "selected.ts", kind: "file" }), + ); + expect(imported.value.entries).toContainEqual( + expect.objectContaining({ path: "caller-keep.log", kind: "file" }), + ); + expect(imported.value.exclusions).not.toContainEqual( + expect.objectContaining({ path: "selected.ts" }), + ); + }); + }, +); diff --git a/tests/boundary/mcp/applicationEvidenceMcp.test.ts b/tests/boundary/mcp/applicationEvidenceMcp.test.ts index 6780c9d01..16dfc142c 100644 --- a/tests/boundary/mcp/applicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/applicationEvidenceMcp.test.ts @@ -1,6 +1,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import type { Client } from "@modelcontextprotocol/client"; import { describe, expect, it } from "vitest"; +import { Ajv2020 } from "ajv/dist/2020.js"; import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { @@ -13,6 +14,7 @@ import { createApplicationMcpHarness, type ApplicationMcpHarness, } from "../../fixtures/applicationMcpHarness.js"; +import { createHistoricalSourceGraph } from "../../../src/domain/referenceSourceGraph.js"; async function runInlineWorkflowScenarios( harness: ApplicationMcpHarness, @@ -89,6 +91,57 @@ async function runInlineWorkflowScenarios( ], }, }); + + const referenceWithExclusion = createHistoricalSourceGraph({ + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + root_sha256: undefined, + inventory_state: "partial", + exclusions: [ + { + path: "vendor/removed.ts", + reason: "project-ignored", + pattern: "vendor/", + }, + ], + }); + const argumentsWithExclusion = { + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, + reference: referenceWithExclusion, + }; + const advertised = (await harness.client.listTools()).tools.find( + ({ name }) => name === "compare_source_to_bundle", + ); + if (advertised === undefined) throw new Error("Comparison tool is missing"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + expect(ajv.validateSchema(advertised.inputSchema)).toBe(true); + const validate = ajv.compile(advertised.inputSchema); + expect(validate(argumentsWithExclusion)).toBe(true); + const missingPattern = { + ...argumentsWithExclusion, + reference: { + ...referenceWithExclusion, + exclusions: [{ path: "vendor/removed.ts", reason: "project-ignored" }], + }, + }; + expect(validate(missingPattern)).toBe(false); + const rejectedExclusion = await harness.client.callTool({ + name: "compare_source_to_bundle", + arguments: missingPattern, + }); + expect(rejectedExclusion.isError).toBe(true); + const sourceComparedWithExclusion = await harness.client.callTool({ + name: "compare_source_to_bundle", + arguments: argumentsWithExclusion, + }); + expect(sourceComparedWithExclusion.isError).not.toBe(true); + expect(sourceComparedWithExclusion.structuredContent).toMatchObject({ + normalized_result: { + reference: { + root_sha256: referenceWithExclusion.root_sha256, + inventory_state: "partial", + }, + }, + }); expect(harness.session.exportEvidenceBundle().records.length).toBeGreaterThan( 2, );