feat(cli): summarize committed reference-source inventories (#2768)
This commit is contained in:
+28
@@ -214,8 +214,36 @@ Import an older source tree to compare with the current artifact:
|
||||
|
||||
```bash
|
||||
rea import-reference-source /absolute/path/to/source
|
||||
rea import-reference-source /absolute/path/to/source --detail summary --outline-depth 2 --json
|
||||
```
|
||||
|
||||
The default (`--detail full`) returns the complete `HistoricalSourceGraph`.
|
||||
Choose `--detail summary` for a `HistoricalSourceSummary` of that same committed
|
||||
inventory: its `HistoricalSourceManifest` contains the full graph and root
|
||||
digests, with provenance, VCS facts, languages, manifest paths, entry and
|
||||
relationship counts, parse-failure counts, file/directory states, exclusions
|
||||
and limitations. The summary is a presentation view, not a graph accepted by
|
||||
source-to-bundle comparison; keep the full output for that workflow.
|
||||
|
||||
`--outline-depth` requires `--detail summary` and accepts a nonnegative safe
|
||||
integer. Depth 1 shows root children, depth 2 (the default) also shows their
|
||||
children, and depth 0 omits all outline entries. Paths retain canonical Unicode
|
||||
code-point order. `outline.omitted_entry_count` counts only inventoried entries
|
||||
hidden by the chosen depth; it does not count unexamined children of excluded or
|
||||
unreadable directories, or change `inventory_state`. Exclusions and source
|
||||
limitations remain separate from this presentation omission.
|
||||
|
||||
`file_sizes.known_bytes` is the exact sum of known inventoried file sizes as
|
||||
decimal text, so large sums cannot lose integer precision. Unknown sizes count
|
||||
in `file_sizes.unknown_file_count`; excluded or unexamined paths are not included
|
||||
in that total. A zero known-byte sum therefore does not establish an empty source
|
||||
tree. Full output retains individual file hashes, relationships and parse-failure
|
||||
diagnostics. Summary mode performs the same import and cleanup, then projects
|
||||
the committed result without rereading source files. It reduces the selected
|
||||
output, not the amount of source examined. A shallow tree or many exclusions can
|
||||
still produce a large summary; depth 0 removes the outline but preserves the
|
||||
other facts. No fixed output-size or runtime improvement is promised.
|
||||
|
||||
The import records hashes and metadata for the supplied files, separately
|
||||
from observations of the current app. For a directory inside a Git checkout,
|
||||
it records the HEAD of the nearest enclosing repository; whether the working
|
||||
|
||||
@@ -2,6 +2,7 @@ import { z } from "incur";
|
||||
|
||||
import type { DirectAnalysis } from "../composition/directAnalysis.js";
|
||||
import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";
|
||||
import { createHistoricalSourceSummary } from "../domain/referenceSourceSummary.js";
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { importReferenceSource } from "../application/ReferenceSourceImport.js";
|
||||
@@ -241,11 +242,48 @@ const registerReferenceSourceCommand = (
|
||||
.string()
|
||||
.describe("Local source directory to import as historical reference"),
|
||||
}),
|
||||
run: ({ args }) =>
|
||||
options: z.object({
|
||||
detail: z
|
||||
.enum(["full", "summary"])
|
||||
.default("full")
|
||||
.describe(
|
||||
"Complete committed graph or compact historical-source summary",
|
||||
),
|
||||
outlineDepth: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.safe()
|
||||
.optional()
|
||||
.describe(
|
||||
"Summary outline path depth (default: 2; 0 omits entries); requires --detail summary",
|
||||
),
|
||||
}),
|
||||
run: ({ args, options }) =>
|
||||
logCliCommand(
|
||||
logger,
|
||||
"import-reference-source",
|
||||
async () => {
|
||||
if (
|
||||
options.outlineDepth !== undefined &&
|
||||
options.detail !== "summary"
|
||||
)
|
||||
return {
|
||||
error: "Import failed",
|
||||
...projectAnalysisError(
|
||||
new AnalysisInputError(
|
||||
"import-reference-source",
|
||||
{ userMessage: "--outline-depth requires --detail summary" },
|
||||
[
|
||||
{
|
||||
path: ["outlineDepth"],
|
||||
reason: "invalid_value",
|
||||
message: "--outline-depth requires --detail summary",
|
||||
},
|
||||
],
|
||||
),
|
||||
),
|
||||
};
|
||||
const config = parseConfig(environment);
|
||||
if (!config.ok)
|
||||
return {
|
||||
@@ -255,6 +293,9 @@ const registerReferenceSourceCommand = (
|
||||
return runReferenceSourceImport(
|
||||
args.root,
|
||||
config.value.referenceSourcePolicy,
|
||||
options.detail === "summary"
|
||||
? (options.outlineDepth ?? 2)
|
||||
: undefined,
|
||||
);
|
||||
},
|
||||
isReferenceSourceImportCliFailure,
|
||||
@@ -265,6 +306,7 @@ const registerReferenceSourceCommand = (
|
||||
const runReferenceSourceImport = async (
|
||||
root: string,
|
||||
policy: ReferenceSourcePolicy,
|
||||
outlineDepth: number | undefined,
|
||||
): Promise<unknown> => {
|
||||
const resources = new ArtifactResourceScope();
|
||||
let outcome:
|
||||
@@ -346,7 +388,9 @@ const runReferenceSourceImport = async (
|
||||
cause: closeOutcome.cause,
|
||||
}),
|
||||
};
|
||||
return imported.value;
|
||||
return outlineDepth === undefined
|
||||
? imported.value
|
||||
: createHistoricalSourceSummary(imported.value, outlineDepth);
|
||||
};
|
||||
|
||||
const mergeReferenceSourceCleanup = (
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createHistoricalSourceGraph,
|
||||
createHistoricalSourceManifest,
|
||||
type HistoricalSourceGraphInput,
|
||||
} from "./referenceSourceGraph.js";
|
||||
import { createHistoricalSourceSummary } from "./referenceSourceSummary.js";
|
||||
|
||||
const input = (
|
||||
entries: HistoricalSourceGraphInput["entries"],
|
||||
): HistoricalSourceGraphInput => ({
|
||||
schema: "HistoricalSourceGraph",
|
||||
authority: "historical-reference",
|
||||
root_alias: "$REFERENCE_ROOT",
|
||||
inventory_state: "unknown",
|
||||
entries,
|
||||
relationships: [],
|
||||
parse_failures: [],
|
||||
exclusions: [],
|
||||
languages: [],
|
||||
manifests: [],
|
||||
vcs: null,
|
||||
provenance: { importer: "fixture", importer_version: null, caller: "test" },
|
||||
limitations: ["Inventory coverage is unknown"],
|
||||
});
|
||||
|
||||
describe("committed historical-source summaries", () => {
|
||||
it("preserves unknown coverage, sizes, directories and missing symlinks without mutating the graph", () => {
|
||||
const graph = createHistoricalSourceGraph(
|
||||
input([
|
||||
{
|
||||
path: "a",
|
||||
kind: "directory",
|
||||
tree_state: "unknown",
|
||||
classifications: ["unknown"],
|
||||
limitations: ["Not enumerated"],
|
||||
},
|
||||
{
|
||||
path: "b",
|
||||
kind: "file",
|
||||
sha256: null,
|
||||
size: null,
|
||||
content_state: "unreadable",
|
||||
language: null,
|
||||
classifications: ["unknown"],
|
||||
limitations: ["Not readable"],
|
||||
},
|
||||
{
|
||||
path: "c",
|
||||
kind: "symlink",
|
||||
target: "missing",
|
||||
target_state: "missing",
|
||||
classifications: ["unknown"],
|
||||
limitations: [],
|
||||
},
|
||||
]),
|
||||
);
|
||||
const original = structuredClone(graph);
|
||||
const summary = createHistoricalSourceSummary(graph, 1);
|
||||
expect(summary).toMatchObject({
|
||||
manifest: createHistoricalSourceManifest(original),
|
||||
inventory_state: "unknown",
|
||||
vcs: null,
|
||||
provenance: original.provenance,
|
||||
file_sizes: { known_bytes: "0", unknown_file_count: 1 },
|
||||
counts: { entries: 3, files: 1, directories: 1, symlinks: 1 },
|
||||
directory_tree_states: { unknown: 1, enumerated: 0 },
|
||||
file_content_states: { unreadable: 1, hashed: 0 },
|
||||
limitations: original.limitations,
|
||||
outline: {
|
||||
entries: [
|
||||
original.entries[0],
|
||||
expect.objectContaining({ size: null, content_state: "unreadable" }),
|
||||
original.entries[2],
|
||||
],
|
||||
omitted_entry_count: 0,
|
||||
},
|
||||
});
|
||||
expect(summary.outline.entries[1]).not.toHaveProperty("sha256");
|
||||
expect(graph).toEqual(original);
|
||||
});
|
||||
|
||||
it("sums known bytes losslessly beyond the JSON safe integer range", () => {
|
||||
const graph = createHistoricalSourceGraph(
|
||||
input(
|
||||
["a", "b"].map((path) => ({
|
||||
path,
|
||||
kind: "file",
|
||||
sha256: "a".repeat(64),
|
||||
size: Number.MAX_SAFE_INTEGER,
|
||||
content_state: "hashed",
|
||||
language: null,
|
||||
classifications: ["unknown"],
|
||||
limitations: [],
|
||||
})),
|
||||
),
|
||||
);
|
||||
expect(createHistoricalSourceSummary(graph, 0)).toMatchObject({
|
||||
file_sizes: { known_bytes: "18014398509481982", unknown_file_count: 0 },
|
||||
outline: { entries: [], omitted_entry_count: 2 },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import {
|
||||
createHistoricalSourceManifest,
|
||||
type HistoricalSourceGraph,
|
||||
} from "./referenceSourceGraph.js";
|
||||
|
||||
type Entry = HistoricalSourceGraph["entries"][number];
|
||||
|
||||
const outlineEntry = (entry: Entry) => {
|
||||
if (entry.kind !== "file") return entry;
|
||||
return {
|
||||
path: entry.path,
|
||||
kind: entry.kind,
|
||||
language: entry.language,
|
||||
size: entry.size,
|
||||
content_state: entry.content_state,
|
||||
classifications: entry.classifications,
|
||||
limitations: entry.limitations,
|
||||
};
|
||||
};
|
||||
|
||||
/** Project one committed inventory without rereading or reinterpreting source. */
|
||||
export const createHistoricalSourceSummary = (
|
||||
graph: HistoricalSourceGraph,
|
||||
outlineDepth: number,
|
||||
) => {
|
||||
const manifest = createHistoricalSourceManifest(graph);
|
||||
const counts = {
|
||||
entries: graph.entries.length,
|
||||
files: 0,
|
||||
directories: 0,
|
||||
symlinks: 0,
|
||||
relationships: graph.relationships.length,
|
||||
parse_failures: graph.parse_failures.length,
|
||||
exclusions: graph.exclusions.length,
|
||||
};
|
||||
const contentStates = {
|
||||
hashed: 0,
|
||||
"redacted-secret": 0,
|
||||
excluded: 0,
|
||||
unreadable: 0,
|
||||
unknown: 0,
|
||||
};
|
||||
const treeStates = {
|
||||
enumerated: 0,
|
||||
partial: 0,
|
||||
excluded: 0,
|
||||
unreadable: 0,
|
||||
unknown: 0,
|
||||
};
|
||||
let knownBytes = 0n;
|
||||
let unknownSizeFiles = 0;
|
||||
const outline: ReturnType<typeof outlineEntry>[] = [];
|
||||
for (const entry of graph.entries) {
|
||||
if (entry.kind === "file") {
|
||||
counts.files += 1;
|
||||
contentStates[entry.content_state] += 1;
|
||||
if (entry.size === null) unknownSizeFiles += 1;
|
||||
else knownBytes += BigInt(entry.size);
|
||||
} else if (entry.kind === "directory") {
|
||||
counts.directories += 1;
|
||||
treeStates[entry.tree_state] += 1;
|
||||
} else counts.symlinks += 1;
|
||||
// Canonical entries already use Unicode code-point order; filtering preserves it.
|
||||
if (entry.path.split("/").length <= outlineDepth)
|
||||
outline.push(outlineEntry(entry));
|
||||
}
|
||||
return {
|
||||
schema: "HistoricalSourceSummary" as const,
|
||||
authority: graph.authority,
|
||||
root_alias: graph.root_alias,
|
||||
manifest,
|
||||
inventory_state: graph.inventory_state,
|
||||
provenance: graph.provenance,
|
||||
vcs: graph.vcs,
|
||||
languages: graph.languages,
|
||||
manifests: graph.manifests,
|
||||
counts,
|
||||
file_content_states: contentStates,
|
||||
directory_tree_states: treeStates,
|
||||
file_sizes: {
|
||||
// Decimal text keeps large totals exact; this covers only known inventoried sizes.
|
||||
known_bytes: knownBytes.toString(),
|
||||
unknown_file_count: unknownSizeFiles,
|
||||
},
|
||||
exclusions: graph.exclusions,
|
||||
limitations: graph.limitations,
|
||||
outline: {
|
||||
depth: outlineDepth,
|
||||
entries: outline,
|
||||
omitted_entry_count: graph.entries.length - outline.length,
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,190 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect } from "vitest";
|
||||
|
||||
import {
|
||||
createHistoricalSourceManifest,
|
||||
parseHistoricalSourceGraph,
|
||||
} from "../../../src/domain/referenceSourceGraph.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
describe.skipIf(process.platform === "win32")(
|
||||
"compiled historical-source summaries",
|
||||
() => {
|
||||
cliTest(
|
||||
"selects a committed summary without changing complete output",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory("rea-reference-summary-");
|
||||
await mkdir(join(root, "src", "nested"), { recursive: true });
|
||||
await Promise.all([
|
||||
writeFile(join(root, "package.json"), '{"name":"fixture"}\n'),
|
||||
writeFile(
|
||||
join(root, "src", "main.js"),
|
||||
'import "./nested/value.js";\n',
|
||||
),
|
||||
writeFile(
|
||||
join(root, "src", "nested", "value.js"),
|
||||
"export const value = 1;\n",
|
||||
),
|
||||
writeFile(join(root, "\uE000.js"), "export const first = 1;\n"),
|
||||
writeFile(join(root, "\u{10000}.js"), "export const last = 2;\n"),
|
||||
]);
|
||||
const run = (...options: string[]) =>
|
||||
cli.run({
|
||||
arguments: ["import-reference-source", root, "--json", ...options],
|
||||
cwd: root,
|
||||
});
|
||||
const complete = await run();
|
||||
expect(complete.exitCode).toBe(0);
|
||||
const graph = parseHistoricalSourceGraph(complete.json);
|
||||
expect((await run("--detail", "full")).stdout).toBe(complete.stdout);
|
||||
for (const depth of [undefined, 0, 1, 10]) {
|
||||
const result = await run(
|
||||
"--detail",
|
||||
"summary",
|
||||
...(depth === undefined ? [] : ["--outline-depth", String(depth)]),
|
||||
);
|
||||
expect(result.exitCode).toBe(0);
|
||||
const selected = graph.entries.filter(
|
||||
(entry) => entry.path.split("/").length <= (depth ?? 2),
|
||||
);
|
||||
expect(result.json).toMatchObject({
|
||||
schema: "HistoricalSourceSummary",
|
||||
authority: graph.authority,
|
||||
root_alias: graph.root_alias,
|
||||
manifest: createHistoricalSourceManifest(graph),
|
||||
inventory_state: graph.inventory_state,
|
||||
provenance: graph.provenance,
|
||||
vcs: graph.vcs,
|
||||
languages: graph.languages,
|
||||
manifests: graph.manifests,
|
||||
counts: {
|
||||
entries: 7,
|
||||
files: 5,
|
||||
directories: 2,
|
||||
symlinks: 0,
|
||||
relationships: 1,
|
||||
parse_failures: 0,
|
||||
exclusions: 0,
|
||||
},
|
||||
outline: {
|
||||
depth: depth ?? 2,
|
||||
entries: selected.map(({ path, kind }) =>
|
||||
expect.objectContaining({ path, kind }),
|
||||
),
|
||||
omitted_entry_count: graph.entries.length - selected.length,
|
||||
},
|
||||
});
|
||||
expect(result.json).not.toHaveProperty("relationships");
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"retains exclusions and parse coverage independently of a hidden outline",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory(
|
||||
"rea-reference-partial-summary-",
|
||||
);
|
||||
await Promise.all([
|
||||
writeFile(join(root, ".gitignore"), "ignored.txt\n"),
|
||||
writeFile(join(root, "ignored.txt"), "ignored\n"),
|
||||
writeFile(join(root, "broken.js"), "export const = ;\n"),
|
||||
]);
|
||||
const result = await cli.run({
|
||||
arguments: [
|
||||
"import-reference-source",
|
||||
root,
|
||||
"--detail",
|
||||
"summary",
|
||||
"--outline-depth",
|
||||
"0",
|
||||
"--json",
|
||||
],
|
||||
cwd: root,
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.json).toMatchObject({
|
||||
inventory_state: "partial",
|
||||
counts: { parse_failures: 1, exclusions: 1 },
|
||||
exclusions: [
|
||||
{
|
||||
path: "ignored.txt",
|
||||
reason: "project-ignored",
|
||||
pattern: "ignored.txt",
|
||||
},
|
||||
],
|
||||
outline: { depth: 0, entries: [], omitted_entry_count: 2 },
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
describe("compiled historical-source summary input boundaries", () => {
|
||||
cliTest(
|
||||
"documents detail and depth in public command help",
|
||||
async ({ cli }) => {
|
||||
const result = await cli.run({
|
||||
arguments: ["import-reference-source", "--help"],
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stdout).toContain("--detail");
|
||||
expect(result.stdout).toContain("--outline-depth");
|
||||
expect(result.stdout).toContain("requires --detail summary");
|
||||
},
|
||||
);
|
||||
|
||||
for (const options of [
|
||||
["--detail", "summry"],
|
||||
["--detail", "summary", "--outline-depth", "-1"],
|
||||
["--detail", "summary", "--outline-depth", "1.5"],
|
||||
["--detail", "summary", "--outline-depth", "NaN"],
|
||||
["--detail", "summary", "--outline-depth", "9007199254740992"],
|
||||
["--outline-depth", "1"],
|
||||
["--detail", "full", "--outline-depth", "1"],
|
||||
]) {
|
||||
cliTest(`rejects ${options.join(" ")}`, async ({ cli }) => {
|
||||
const root = await createTestTempDirectory(
|
||||
"rea-reference-summary-input-",
|
||||
);
|
||||
const result = await cli.run({
|
||||
arguments: ["import-reference-source", root, ...options, "--json"],
|
||||
cwd: root,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.stdout).not.toContain("HistoricalSourceGraph");
|
||||
expect(result.stdout).not.toContain("HistoricalSourceSummary");
|
||||
expect(result.stdout).toMatch(/detail|outline/u);
|
||||
});
|
||||
}
|
||||
|
||||
cliTest(
|
||||
"preserves the missing-root failure in summary mode",
|
||||
async ({ cli }) => {
|
||||
const root = await createTestTempDirectory(
|
||||
"rea-reference-summary-missing-",
|
||||
);
|
||||
const result = await cli.run({
|
||||
arguments: [
|
||||
"import-reference-source",
|
||||
join(root, "missing"),
|
||||
"--detail",
|
||||
"summary",
|
||||
"--json",
|
||||
],
|
||||
cwd: root,
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.json).toMatchObject({
|
||||
error: "Import failed",
|
||||
category: "invalid_input",
|
||||
message: expect.stringContaining(
|
||||
"Reference source root could not be resolved",
|
||||
),
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user