Implement workspace evolution, migrations, and runtime continuity
Enable evolution by default for source-backed workspaces. Add stable conformance ownership, semantic-major review, candidate typechecking, and durable fenced cutover with explicit migrations and forward recovery. Independently supervise package runtimes so unchanged resource owners keep their processes and connections across cutover. Add scoped invocation authority, resource sessions, and typed callback rebinding. Wire opaque object references through generated bindings and RPCs. Add canonical relationship sets, keyed maps, and ordered lists with scoped transactional mutations, revision checks, and inverse consistency. Support planned cascade deletion, protection, tombstones, and lifecycle foundations. Add journaled structural edits, package/function/migration scaffolding, managed repository creation, and resumable bottom-up dependency pin publication. Document lifetime boundaries, revision pinning, prototype compatibility policy, commands, and deferred work. Validate with 210 tests, user-systemd process/connection continuity, generated-package TypeScript checks, and Nix host/protocol checks. TTL handoff, physical reclamation, general multi-step migrations, and root-systemd migration isolation acceptance remain deferred.
This commit is contained in:
Timothy J. Aveni
@@ -0,0 +1,179 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { createHash } from "node:crypto";
|
||||
import { compileWorkspaceRepository, compileCapabilityResourceRepository } from "./assembly.js";
|
||||
import { createGitCapabilityResolver } from "./git-resolver.js";
|
||||
import { contentDigest, planEvolution, type EvolutionReview, type WorkspaceRevision } from "../capability-model/index.js";
|
||||
import { bindingSchema, generateTypeScriptBindings, type BindingSchema, type TypeScriptBindingOptions } from "../bindings/index.js";
|
||||
const execFile = promisify(execFileCallback);
|
||||
const bytesDigest = (value: Uint8Array) => `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
|
||||
export const localResourceSnapshots = async (root: string, filename?: string): Promise<{resources: {kind: string; repository: string; commit: string; directory: string}[]}> => {
|
||||
if (filename) {
|
||||
const document = JSON.parse(await fs.readFile(filename, "utf8"));
|
||||
return {resources: document.resources.map((entry: {directory: string}) => ({...entry, directory: path.resolve(path.dirname(filename), entry.directory)}))};
|
||||
}
|
||||
let directory = await fs.realpath(root);
|
||||
for (;;) {
|
||||
let graphText: string | undefined;
|
||||
try {graphText = await fs.readFile(path.join(directory, ".quixos/resource-graph.json"), "utf8");}
|
||||
catch (error) {if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;}
|
||||
if (graphText !== undefined) {
|
||||
const graph = JSON.parse(graphText);
|
||||
const resources = [];
|
||||
for (const entry of graph.resources) {
|
||||
const location = await fs.realpath(path.resolve(directory, entry.directory));
|
||||
if (!location.startsWith(`${directory}/resources/`)) throw new Error("Workbench resource escapes managed directory");
|
||||
resources.push({kind: entry.kind, ...entry.source, directory: location});
|
||||
}
|
||||
return {resources};
|
||||
}
|
||||
const parent = path.dirname(directory);
|
||||
if (parent === directory) return {resources: []};
|
||||
directory = parent;
|
||||
}
|
||||
};
|
||||
|
||||
/** Copy actual authoring files without snapshotting jj or creating a Git commit. */
|
||||
export const snapshotRepository = async (source: string, destination: string) => {
|
||||
const root = await fs.realpath(source);
|
||||
const files = async () => (await execFile("git", ["-C", root, "ls-files", "--cached", "--others", "--exclude-standard", "-z"], { maxBuffer: 16 * 1024 * 1024 })).stdout.split("\0").filter(Boolean).sort();
|
||||
const names = [...new Set(await files())];
|
||||
if (names.length > 50_000) throw new Error("Candidate source exceeds 50000 files");
|
||||
const contents: {name: string; digest: string; mode: number}[] = [];
|
||||
let bytes = 0;
|
||||
await fs.mkdir(destination, { recursive: true, mode: 0o700 });
|
||||
for (const name of names) {
|
||||
if (path.isAbsolute(name) || name.split(/[\\/]/).some((part) => part === ".." || part === ".git" || part === ".jj")) throw new Error("Invalid candidate source path");
|
||||
const file = path.join(root, name);
|
||||
let metadata;
|
||||
try { metadata = await fs.lstat(file); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") continue; throw error; }
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || !(await fs.realpath(file)).startsWith(`${root}${path.sep}`)) throw new Error(`Candidate source must be a regular file: ${name}`);
|
||||
const data = await fs.readFile(file);
|
||||
bytes += data.length;
|
||||
if (bytes > 128 * 1024 * 1024) throw new Error("Candidate source exceeds 128 MiB");
|
||||
contents.push({ name, digest: bytesDigest(data), mode: metadata.mode & 0o777 });
|
||||
await fs.mkdir(path.dirname(path.join(destination, name)), { recursive: true });
|
||||
await fs.writeFile(path.join(destination, name), data, { flag: "wx", mode: metadata.mode & 0o777 });
|
||||
}
|
||||
if (JSON.stringify([...new Set(await files())]) !== JSON.stringify(names)) throw new Error("Source files changed during candidate snapshot");
|
||||
for (const entry of contents) if (bytesDigest(await fs.readFile(path.join(root, entry.name))) !== entry.digest) throw new Error(`Source changed during candidate snapshot: ${entry.name}`);
|
||||
return { source: root, directory: destination, treeDigest: contentDigest(contents), files: contents };
|
||||
};
|
||||
|
||||
export const checkResourceCandidate = async (options: {root: string; output: string; kind: "package" | "interface"; source: {repository: string; commit: string}; snapshotMap?: string; publishedOnly?: boolean}) => {
|
||||
await fs.mkdir(options.output, {mode: 0o700});
|
||||
const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "qx-resource-check-"));
|
||||
const blockers: string[] = [];
|
||||
let diagnostics = "";
|
||||
let treeDigest: string | undefined;
|
||||
try {
|
||||
const root = await snapshotRepository(options.root, path.join(temporary, "root"));
|
||||
treeDigest = root.treeDigest;
|
||||
const map = options.publishedOnly ? {resources: []} : await localResourceSnapshots(options.root, options.snapshotMap);
|
||||
const resources = [];
|
||||
for (const [index, entry] of map.resources.entries()) {
|
||||
const snapshot = await snapshotRepository(entry.directory, path.join(temporary, `dependency-${index}`));
|
||||
resources.push({...entry, directory: snapshot.directory});
|
||||
}
|
||||
const snapshotMap = path.join(temporary, "snapshots.json");
|
||||
await fs.writeFile(snapshotMap, JSON.stringify({resources}));
|
||||
const resolveResource = await createGitCapabilityResolver({checkoutRoot: path.join(temporary, "resolved"), snapshotMap});
|
||||
const compiled = await compileCapabilityResourceRepository({rootDirectory: root.directory, kind: options.kind, source: {resolver: "git", ...options.source}, resolveResource});
|
||||
if (compiled.resource.kind === "package") {
|
||||
const configuration = JSON.parse(await fs.readFile(path.join(root.directory, "quixos.check.json"), "utf8"));
|
||||
const output = configuration.bindingOutput as string;
|
||||
if (configuration.backend !== "typescript" || !/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.ts$/.test(output)) throw new Error("Unsupported candidate checker configuration");
|
||||
const modules = path.join(root.source, "node_modules");
|
||||
await fs.access(path.join(modules, ".bin/tsc"));
|
||||
await fs.symlink(modules, path.join(root.directory, "node_modules"), "dir");
|
||||
const destination = path.join(root.directory, output);
|
||||
await fs.mkdir(path.dirname(destination), {recursive: true});
|
||||
await fs.writeFile(destination, generateTypeScriptBindings(bindingSchema(compiled), compiled.resource.revision.revisionId, configuration.options));
|
||||
diagnostics = (await execFile(path.join(modules, ".bin/tsc"), ["--noEmit", "--pretty", "false"], {cwd: root.directory, maxBuffer: 16 * 1024 * 1024})).stdout;
|
||||
}
|
||||
await fs.writeFile(path.join(options.output, "candidate.json"), JSON.stringify(compiled.resource, null, 2));
|
||||
} catch (error) {
|
||||
blockers.push(error instanceof Error ? error.message : String(error));
|
||||
diagnostics += (error as {stdout?: string; stderr?: string}).stdout ?? "";
|
||||
diagnostics += (error as {stderr?: string}).stderr ?? "";
|
||||
} finally {await fs.rm(temporary, {recursive: true, force: true});}
|
||||
const result = {candidateOnly: true, activationEvidence: false, treeDigest, blockers, diagnostics};
|
||||
await fs.writeFile(path.join(options.output, "report.json"), JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
};
|
||||
|
||||
export const checkWorkspaceCandidate = async (options: { root: string; output: string; snapshotMap?: string; baseline?: string; reviews?: string }) => {
|
||||
// A new output directory is the whole artifact boundary; never overwrite a prior check.
|
||||
await fs.mkdir(options.output, { mode: 0o700 });
|
||||
const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "quixos-candidate-"));
|
||||
const blockers: string[] = [];
|
||||
const checks: unknown[] = [];
|
||||
const snapshots = [];
|
||||
try {
|
||||
const root = await snapshotRepository(options.root, path.join(temporary, "root"));
|
||||
snapshots.push(root);
|
||||
const map = await localResourceSnapshots(options.root, options.snapshotMap);
|
||||
const resources = [];
|
||||
for (const [index, entry] of map.resources.entries()) {
|
||||
const source = entry.directory;
|
||||
const snapshot = await snapshotRepository(source, path.join(temporary, `resource-${index}`));
|
||||
snapshots.push(snapshot);
|
||||
resources.push({ ...entry, directory: snapshot.directory });
|
||||
}
|
||||
const mapFile = path.join(temporary, "snapshots.json");
|
||||
await fs.writeFile(mapFile, JSON.stringify({ resources }));
|
||||
const resolveResource = await createGitCapabilityResolver({ checkoutRoot: path.join(temporary, "resolved"), snapshotMap: mapFile });
|
||||
const compiled = await compileWorkspaceRepository({ rootDirectory: root.directory, resolveResource });
|
||||
const baseline = options.baseline ? JSON.parse(await fs.readFile(options.baseline, "utf8")) as WorkspaceRevision : null;
|
||||
const reviews = options.reviews ? JSON.parse(await fs.readFile(options.reviews, "utf8")) as EvolutionReview[] : [];
|
||||
const evolution = planEvolution(baseline, compiled.workspace, { reviews });
|
||||
blockers.push(...evolution.blockers);
|
||||
const schema: BindingSchema = { format: "quixos-bindings", version: 1, interfaces: compiled.workspace.interfaceImports, packages: compiled.workspace.packageImports };
|
||||
for (const resource of compiled.resources.filter((entry) => entry.kind === "package")) {
|
||||
if (resource.resource.kind !== "package") continue;
|
||||
const revision = resource.resource.revision;
|
||||
let config: { backend: string; bindingOutput: string; options?: TypeScriptBindingOptions };
|
||||
try { config = JSON.parse(await fs.readFile(path.join(resource.directory, "quixos.check.json"), "utf8")); }
|
||||
catch { blockers.push(`No candidate checker configured for ${revision.revisionId} (quixos.check.json)`); continue; }
|
||||
if (config.backend !== "typescript" || !/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.ts$/.test(config.bindingOutput)
|
||||
|| config.bindingOutput.split("/").includes("..")) { blockers.push(`Unsupported checker or binding path for ${revision.revisionId}`); continue; }
|
||||
const sourceSnapshot = snapshots.find((entry) => entry.directory === resource.directory);
|
||||
const dependencyRoot = sourceSnapshot?.source ?? resource.directory;
|
||||
const modules = path.join(dependencyRoot, "node_modules");
|
||||
try { await fs.access(path.join(modules, ".bin", "tsc")); }
|
||||
catch { blockers.push(`Missing installed TypeScript checker/dependencies for ${revision.revisionId}; install its locked development dependencies first`); continue; }
|
||||
if (sourceSnapshot) await fs.symlink(modules, path.join(resource.directory, "node_modules"), "dir");
|
||||
const generated = generateTypeScriptBindings(schema, revision.revisionId, config.options);
|
||||
const destination = path.join(resource.directory, config.bindingOutput);
|
||||
await fs.mkdir(path.dirname(destination), { recursive: true });
|
||||
await fs.writeFile(destination, generated);
|
||||
let success = false, diagnostics = "";
|
||||
try { diagnostics = (await execFile(path.join(modules, ".bin", "tsc"), ["--noEmit", "--pretty", "false", "--listFiles"], { cwd: resource.directory, maxBuffer: 16 * 1024 * 1024 })).stdout; success = true; }
|
||||
catch (error) { const result = error as Error & {stdout?: string; stderr?: string}; diagnostics = `${result.stdout ?? ""}\n${result.stderr ?? result.message}`; }
|
||||
const typeInputs = [];
|
||||
for (const line of diagnostics.split(/\r?\n/)) if (path.isAbsolute(line) && /\.[cm]?tsx?$/.test(line)) {
|
||||
try { typeInputs.push({file: line, digest: bytesDigest(await fs.readFile(line))}); } catch { success = false; }
|
||||
}
|
||||
const checker = await fs.realpath(path.join(modules, ".bin", "tsc"));
|
||||
const check = { packageRevisionId: revision.revisionId, success, bindingSchemaDigest: contentDigest(schema), generatedDigest: contentDigest(generated),
|
||||
checkerDigest: contentDigest({ executable: bytesDigest(await fs.readFile(checker)), typeInputs }), diagnostics };
|
||||
checks.push(check);
|
||||
if (!success) blockers.push(`Typecheck failed for ${revision.revisionId}`);
|
||||
}
|
||||
const result = { schemaVersion: 1, candidateOnly: true, activationEvidence: false,
|
||||
sourceDigest: contentDigest(snapshots.map(({source, treeDigest}) => ({source, treeDigest}))),
|
||||
snapshots: snapshots.map(({source, treeDigest}) => ({source, treeDigest})), evolution, checks, blockers,
|
||||
note: "Local source-tree checks do not certify old Git revisions. Publication must repin the DAG and recheck final immutable artifacts." };
|
||||
await fs.writeFile(path.join(options.output, "candidate.json"), JSON.stringify(compiled.workspace, null, 2));
|
||||
await fs.writeFile(path.join(options.output, "report.json"), JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
} catch (error) {
|
||||
const result = { schemaVersion: 1, candidateOnly: true, activationEvidence: false, checks, blockers: [...blockers, error instanceof Error ? error.message : String(error)] };
|
||||
await fs.writeFile(path.join(options.output, "report.json"), JSON.stringify(result, null, 2));
|
||||
return result;
|
||||
} finally { await fs.rm(temporary, { recursive: true, force: true }); }
|
||||
};
|
||||
Reference in New Issue
Block a user