Implement workspace evolution, migrations, and runtime continuity
Enable evolution by default for source-backed workspaces. Add stable conformance ownership, semantic-major review, candidate typechecking, and durable fenced cutover with explicit migrations and forward recovery. Independently supervise package runtimes so unchanged resource owners keep their processes and connections across cutover. Add scoped invocation authority, resource sessions, and typed callback rebinding. Wire opaque object references through generated bindings and RPCs. Add canonical relationship sets, keyed maps, and ordered lists with scoped transactional mutations, revision checks, and inverse consistency. Support planned cascade deletion, protection, tombstones, and lifecycle foundations. Add journaled structural edits, package/function/migration scaffolding, managed repository creation, and resumable bottom-up dependency pin publication. Document lifetime boundaries, revision pinning, prototype compatibility policy, commands, and deferred work. Validate with 210 tests, user-systemd process/connection continuity, generated-package TypeScript checks, and Nix host/protocol checks. TTL handoff, physical reclamation, general multi-step migrations, and root-systemd migration isolation acceptance remain deferred.
This commit is contained in:
Timothy J. Aveni
@@ -0,0 +1,189 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { contentDigest } from "../capability-model/evolution.js";
|
||||
import { editStructure, type StructuralEdit } from "./structural-edits.js";
|
||||
import { snapshotRepository, localResourceSnapshots } from "./candidate-check.js";
|
||||
import { compileWorkspaceRepository, compileCapabilityResourceRepository } from "./assembly.js";
|
||||
import { createGitCapabilityResolver } from "./git-resolver.js";
|
||||
import {bindingSchema, generateTypeScriptBindings} from "../bindings/index.js";
|
||||
|
||||
export type StructuralRequest = {
|
||||
kind: "workspace" | "interface" | "package";
|
||||
source?: {repository: string; commit: string};
|
||||
resourceRoot?: string;
|
||||
files: ({file: string; edits: StructuralEdit[]} | {file: string; create: string} | {file: string; generated: string})[];
|
||||
};
|
||||
type Change = {file: string; before: string | null; after: string; mode: number};
|
||||
type Journal = {schemaVersion: 1; id: string; root: string; phase: "prepared" | "complete"; changes: Change[]};
|
||||
const safeFile = (file: string) => {
|
||||
if (!/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*(?:\.gitignore|\.yarnrc.yml|[A-Za-z0-9_-][A-Za-z0-9_.-]*\.(?:qx|lock|ts|tsx|json|nix|txtpb))$/.test(file)
|
||||
|| file.split("/").some((part) => [".git", ".jj", ".quixos", "node_modules"].includes(part))) throw new Error(`Unsafe scaffold path ${file}`);
|
||||
};
|
||||
const read = async (root: string, file: string): Promise<string | null> => {
|
||||
safeFile(file);
|
||||
const target = path.join(root, file);
|
||||
try {
|
||||
const metadata = await fs.lstat(target);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || !(await fs.realpath(target)).startsWith(`${root}/`)) throw new Error(`Scaffold target is not a contained regular file: ${file}`);
|
||||
return await fs.readFile(target, "utf8");
|
||||
} catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; }
|
||||
};
|
||||
const containedParent = async (root: string, file: string) => {
|
||||
let current = root;
|
||||
for (const part of file.split("/").slice(0, -1)) {
|
||||
current = path.join(current, part);
|
||||
await fs.mkdir(current).catch((error: NodeJS.ErrnoException) => { if (error.code !== "EEXIST") throw error; });
|
||||
const metadata = await fs.lstat(current);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Scaffold parent must be a real directory");
|
||||
}
|
||||
};
|
||||
const durableJson = async (file: string, value: unknown) => {
|
||||
const temporary = `${file}.${randomUUID()}.tmp`;
|
||||
const handle = await fs.open(temporary, "wx", 0o600);
|
||||
try { await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`); await handle.sync(); } finally { await handle.close(); }
|
||||
await fs.rename(temporary, file);
|
||||
const directory = await fs.open(path.dirname(file), "r");
|
||||
try { await directory.sync(); } finally { await directory.close(); }
|
||||
};
|
||||
|
||||
/** Validate the entire edited resource graph in a private snapshot before writes. */
|
||||
export const planStructure = async (rootPath: string, request: StructuralRequest, snapshotMap?: string) => {
|
||||
const root = await fs.realpath(rootPath);
|
||||
const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "qx-structure-"));
|
||||
try {
|
||||
const snapshot = await snapshotRepository(root, path.join(temporary, "source"));
|
||||
const observed = await Promise.all(snapshot.files.map(async ({name}) => ({file: name, digest: contentDigest(await fs.readFile(path.join(snapshot.directory, name), "utf8"))})));
|
||||
const changes: Change[] = [];
|
||||
if (!Array.isArray(request.files) || !request.files.length || request.files.length > 100) throw new Error("Structural plan requires 1–100 files");
|
||||
for (const input of request.files) {
|
||||
safeFile(input.file);
|
||||
if (changes.some((entry) => entry.file === input.file)) throw new Error("Repeated structural file target");
|
||||
const before = await read(root, input.file);
|
||||
let after: string;
|
||||
if ("create" in input) {
|
||||
if (before !== null || typeof input.create !== "string") throw new Error("Scaffold creation cannot replace an existing file");
|
||||
after = input.create;
|
||||
} else if ("generated" in input) {
|
||||
const generated = (text: string) => text.startsWith("// Generated by qx-scaffold-v1\n") || text.startsWith("# Generated by qx-scaffold-v1\n") || (() => {try {return JSON.parse(text).generatedBy === "qx-scaffold-v1";} catch {return false;}})();
|
||||
if (typeof input.generated !== "string" || !generated(input.generated) || (before !== null && !generated(before))) throw new Error("Only scaffold-owned generated files may be regenerated");
|
||||
after = input.generated;
|
||||
} else {
|
||||
if (before === null || !Array.isArray(input.edits)) throw new Error("Structural edit requires an existing source");
|
||||
after = input.edits.reduce(editStructure, before);
|
||||
}
|
||||
if (Buffer.byteLength(after) > 1024 * 1024) throw new Error("Scaffold file exceeds 1 MiB");
|
||||
const mode = before === null ? 0o644 : (await fs.stat(path.join(root, input.file))).mode & 0o777;
|
||||
changes.push({file: input.file, before, after, mode});
|
||||
await containedParent(snapshot.directory, input.file);
|
||||
await fs.writeFile(path.join(snapshot.directory, input.file), after);
|
||||
}
|
||||
const localMap = path.join(temporary, "local-resources.json");
|
||||
await fs.writeFile(localMap, JSON.stringify(await localResourceSnapshots(root, snapshotMap)));
|
||||
const resolveResource = await createGitCapabilityResolver({checkoutRoot: path.join(temporary, "resources"), snapshotMap: localMap});
|
||||
if (request.resourceRoot && !/^[A-Za-z0-9_-][A-Za-z0-9_.-]*(?:\/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*$/.test(request.resourceRoot)) throw new Error("Resource root must be a contained relative directory");
|
||||
const resourceRoot = path.join(snapshot.directory, request.resourceRoot ?? "");
|
||||
if (request.kind === "workspace") await compileWorkspaceRepository({rootDirectory: resourceRoot, resolveResource});
|
||||
else if (["package", "interface"].includes(request.kind) && request.source) {
|
||||
const compiled = await compileCapabilityResourceRepository({rootDirectory: resourceRoot, kind: request.kind as "package" | "interface", source: {resolver: "git", ...request.source}, resolveResource});
|
||||
let scaffoldOwned = false;
|
||||
try { scaffoldOwned = JSON.parse(await fs.readFile(path.join(resourceRoot, "quixos.scaffold.json"), "utf8")).generatedBy === "qx-scaffold-v1"; } catch { /* ordinary resource, no generated package scaffolding */ }
|
||||
if (scaffoldOwned && compiled.resource.kind === "package") {
|
||||
const configuration = JSON.parse(await fs.readFile(path.join(resourceRoot, "quixos.check.json"), "utf8"));
|
||||
const artifacts = [
|
||||
{file: configuration.bindingOutput as string, after: generateTypeScriptBindings(bindingSchema(compiled), compiled.resource.revision.revisionId, configuration.options)},
|
||||
{file: "quixos.resources.json", after: JSON.stringify({generatedBy: "qx-scaffold-v1", resources: compiled.resources.filter((entry) => entry.directory !== resourceRoot).map((entry) => ({kind: entry.kind, repository: entry.source.repository, commit: entry.source.commit}))}, null, 2) + "\n"},
|
||||
];
|
||||
for (const artifact of artifacts) {
|
||||
const file = request.resourceRoot ? `${request.resourceRoot}/${artifact.file}` : artifact.file;
|
||||
safeFile(file);
|
||||
if (Buffer.byteLength(artifact.after) > 1024 * 1024) throw new Error("Generated scaffold file exceeds 1 MiB");
|
||||
const before = await read(root, file);
|
||||
if (before !== null && !before.startsWith("// Generated by quixos-codegen-ts.") && (() => {try {return JSON.parse(before).generatedBy !== "qx-scaffold-v1";} catch {return true;}})()) throw new Error(`Refusing to overwrite hand-authored generated artifact ${file}`);
|
||||
const previous = changes.find((entry) => entry.file === file);
|
||||
if (previous) previous.after = artifact.after;
|
||||
else changes.push({file, before, after: artifact.after, mode: 0o644});
|
||||
}
|
||||
}
|
||||
}
|
||||
else throw new Error("Resource plans require kind and exact authored source identity");
|
||||
if (changes.length > 100) throw new Error("Structural plan including generated artifacts exceeds 100 files");
|
||||
// Validation may fetch dependencies; reject edits made while it was running.
|
||||
for (const entry of changes) if (await read(root, entry.file) !== entry.before) throw new Error(`Source changed while planning: ${entry.file}`);
|
||||
for (const entry of observed) if (contentDigest(await fs.readFile(path.join(root, entry.file), "utf8")) !== entry.digest) throw new Error(`Validation input changed while planning: ${entry.file}`);
|
||||
return {root, changes, observed, digest: contentDigest(changes), validation: "resource-graph" as const};
|
||||
} finally { await fs.rm(temporary, {recursive: true, force: true}); }
|
||||
};
|
||||
|
||||
/** Replay only exact before/after states. A crash never loses the original text. */
|
||||
const replayStructure = async (rootPath: string, id: string) => {
|
||||
const root = await fs.realpath(rootPath);
|
||||
if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error("Invalid scaffold journal ID");
|
||||
const journalPath = path.join(root, ".quixos", "scaffolds", `${id}.json`);
|
||||
const journal = JSON.parse(await fs.readFile(journalPath, "utf8")) as Journal;
|
||||
if (journal.schemaVersion !== 1 || journal.root !== root || journal.id !== id) throw new Error("Scaffold journal identity mismatch");
|
||||
for (const entry of journal.changes) {
|
||||
const current = await read(root, entry.file);
|
||||
if (current !== entry.before && current !== entry.after) throw new Error(`Scaffold conflicts with newer edits: ${entry.file}; original text is retained in ${journalPath}`);
|
||||
}
|
||||
if (journal.phase === "complete") return {id, journalPath, phase: journal.phase};
|
||||
for (const entry of journal.changes) {
|
||||
if (await read(root, entry.file) === entry.after) continue;
|
||||
await containedParent(root, entry.file);
|
||||
const target = path.join(root, entry.file);
|
||||
const temporary = `${target}.qx-${randomUUID()}.tmp`;
|
||||
const handle = await fs.open(temporary, "wx", entry.mode);
|
||||
try { await handle.writeFile(entry.after); await handle.sync(); } finally { await handle.close(); }
|
||||
if (entry.before === null) {
|
||||
// link is atomic and fails if another author created the destination.
|
||||
await fs.link(temporary, target);
|
||||
await fs.unlink(temporary);
|
||||
} else {
|
||||
if (await read(root, entry.file) !== entry.before) throw new Error(`Source changed during scaffold: ${entry.file}`);
|
||||
await fs.rename(temporary, target);
|
||||
}
|
||||
const directory = await fs.open(path.dirname(target), "r");
|
||||
try { await directory.sync(); } finally { await directory.close(); }
|
||||
}
|
||||
journal.phase = "complete";
|
||||
await durableJson(journalPath, journal);
|
||||
return {id, journalPath, phase: journal.phase};
|
||||
};
|
||||
|
||||
const withStructureLock = async <T>(root: string, work: () => Promise<T>) => {
|
||||
await containedParent(root, ".quixos/scaffolds/placeholder.json");
|
||||
const lock = path.join(root, ".quixos", "scaffolds", "writer.lock");
|
||||
// Never steal a possibly live writer's lock. A process crash requires the
|
||||
// operator to verify that writer is gone, remove this lock, then resume its
|
||||
// journal. This is deliberately fail-closed instead of guessing from a PID.
|
||||
const handle = await fs.open(lock, "wx", 0o600).catch((error) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") throw new Error(`Another scaffold writer or interrupted writer owns ${lock}; verify it has exited before removing its lock and resuming`);
|
||||
throw error;
|
||||
});
|
||||
try { await handle.writeFile(JSON.stringify({pid: process.pid})); await handle.sync(); return await work(); }
|
||||
finally { await handle.close(); await fs.unlink(lock); }
|
||||
};
|
||||
export const resumeStructure = async (rootPath: string, id: string) => {
|
||||
const root = await fs.realpath(rootPath);
|
||||
return withStructureLock(root, () => replayStructure(root, id));
|
||||
};
|
||||
export const applyStructure = async (plan: Awaited<ReturnType<typeof planStructure>>, id: string = randomUUID()) => withStructureLock(plan.root, async () => {
|
||||
if (!/^[a-f0-9-]{36}$/.test(id)) throw new Error("Invalid scaffold journal ID");
|
||||
const directory = path.join(plan.root, ".quixos", "scaffolds");
|
||||
try {
|
||||
const existing = JSON.parse(await fs.readFile(path.join(directory, `${id}.json`), "utf8")) as Journal;
|
||||
if (existing.root !== plan.root || contentDigest(existing.changes) !== plan.digest) throw new Error("Scaffold journal identity conflict");
|
||||
for (const entry of plan.observed) if (!plan.changes.some((change) => change.file === entry.file) && contentDigest(await fs.readFile(path.join(plan.root, entry.file), "utf8")) !== entry.digest) throw new Error(`Stale scaffold validation input: ${entry.file}`);
|
||||
return replayStructure(plan.root, id);
|
||||
} catch (error) {if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;}
|
||||
// An unfinished journal must be recovered before another structural mutation.
|
||||
for (const file of await fs.readdir(directory)) if (file.endsWith(".json")) {
|
||||
const prior = JSON.parse(await fs.readFile(path.join(directory, file), "utf8")) as Journal;
|
||||
if (prior.phase !== "complete") throw new Error(`Unfinished scaffold ${prior.id}; resume it first`);
|
||||
}
|
||||
for (const entry of plan.changes) if (await read(plan.root, entry.file) !== entry.before) throw new Error(`Stale scaffold plan: ${entry.file}`);
|
||||
for (const entry of plan.observed) if (contentDigest(await fs.readFile(path.join(plan.root, entry.file), "utf8")) !== entry.digest) throw new Error(`Stale scaffold validation input: ${entry.file}`);
|
||||
await durableJson(path.join(directory, `${id}.json`), {schemaVersion: 1, id, root: plan.root, phase: "prepared", changes: plan.changes} satisfies Journal);
|
||||
return replayStructure(plan.root, id);
|
||||
});
|
||||
Reference in New Issue
Block a user