Implement workspace evolution, migrations, and runtime continuity

Enable evolution by default for source-backed workspaces. Add stable conformance ownership, semantic-major review, candidate typechecking, and durable fenced cutover with explicit migrations and forward recovery. Independently supervise package runtimes so unchanged resource owners keep their processes and connections across cutover. Add scoped invocation authority, resource sessions, and typed callback rebinding. Wire opaque object references through generated bindings and RPCs. Add canonical relationship sets, keyed maps, and ordered lists with scoped transactional mutations, revision checks, and inverse consistency. Support planned cascade deletion, protection, tombstones, and lifecycle foundations. Add journaled structural edits, package/function/migration scaffolding, managed repository creation, and resumable bottom-up dependency pin publication. Document lifetime boundaries, revision pinning, prototype compatibility policy, commands, and deferred work. Validate with 210 tests, user-systemd process/connection continuity, generated-package TypeScript checks, and Nix host/protocol checks. TTL handoff, physical reclamation, general multi-step migrations, and root-systemd migration isolation acceptance remain deferred.
2026-09-10 18:27:41 -07:00
parent 1e25f391e7
commit 483bc68a94
38 changed files with 3790 additions and 1463 deletions
@@ -0,0 +1,249 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import {randomUUID} from "node:crypto";
+import {execFile as callback} from "node:child_process";
+import {promisify} from "node:util";
+import {loadQuixosLock, parseQuixosLockDocument} from "../resource-lock/index.js";
+import {contentDigest} from "../capability-model/evolution.js";
+import {planStructure, applyStructure, type StructuralRequest} from "./structural-plan.js";
+import {snapshotRepository, checkResourceCandidate, checkWorkspaceCandidate} from "./candidate-check.js";
+import {compileWorkspaceRepository, compileCapabilityResourceRepository, type ResolvedCapabilityResource} from "./assembly.js";
+import {createGitCapabilityResolver} from "./git-resolver.js";
+const execFile = promisify(callback);
+type Source = {repository: string; commit: string};
+export type UpgradeNode = {kind: "workspace" | "package" | "interface"; directory: string; source: Source};
+export type UpgradeSpec = {nodes: UpgradeNode[]; quixos?: Source; baseline?: string; reviews?: string; bootstrap?: boolean};
+type NodePlan = UpgradeNode & {treeDigest: string; dependencies: string[]; lockFiles: string[]};
+export type UpgradePlan = {schemaVersion: 1; workbench: string; spec: UpgradeSpec; nodes: NodePlan[]; digest: string};
+type Step = {directory: string; phase: "editing" | "prepared" | "refactor" | "checked" | "publishing" | "published"; commit?: string; treeDigest?: string; structuralJournal?: string; structuralPlan?: Awaited<ReturnType<typeof planStructure>>};
+type Journal = {schemaVersion: 1; plan: UpgradePlan; steps: Step[]};
+const sourceKey = (node: {kind: string; source: Source}) => JSON.stringify([node.kind, node.source.repository, node.source.commit]);
+const command = async (cwd: string, tool: string, args: string[]) => (await execFile(tool, args, {cwd, maxBuffer: 16 * 1024 * 1024, env: {...process.env, GIT_TERMINAL_PROMPT: "0", QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_SUBTREE_PUBLISH: "0"}})).stdout.trim();
+const validSource = (value: Source) => {
+  const url = new URL(value.repository);
+  if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value.commit)) throw new Error("Upgrade sources must be exact credential-free HTTPS revisions");
+};
+const location = async (root: string, directory: string) => {
+  if (directory !== "root" && !/^resources\/[A-Za-z0-9][A-Za-z0-9._-]*$/.test(directory)) throw new Error("Upgrade target must be a managed root/resource repository");
+  const resolved = await fs.realpath(path.join(root, directory));
+  if (resolved !== path.join(root, directory)) throw new Error("Upgrade target crosses a symlink");
+  return resolved;
+};
+const treeDigest = async (root: string) => {
+  const temporary = await fs.mkdtemp(path.join(os.tmpdir(), "qx-upgrade-tree-"));
+  try {return (await snapshotRepository(root, temporary)).treeDigest;} finally {await fs.rm(temporary, {recursive: true, force: true});}
+};
+const writeJournal = async (filename: string, journal: unknown) => {
+  const temp = `${filename}.${randomUUID()}.tmp`;
+  const handle = await fs.open(temp, "wx", 0o600);
+  try {await handle.writeFile(JSON.stringify(journal, null, 2)); await handle.sync();} finally {await handle.close();}
+  await fs.rename(temp, filename);
+  const directory = await fs.open(path.dirname(filename), "r");
+  try {await directory.sync();} finally {await directory.close();}
+};
+
+export const discoverUpgradeSpec = async (workbench: string): Promise<UpgradeSpec> => {
+  const graph = JSON.parse(await fs.readFile(path.join(workbench, ".quixos/resource-graph.json"), "utf8"));
+  const root = await location(workbench, "root");
+  const nodes: UpgradeNode[] = [{kind: "workspace", directory: "root", source: {
+    repository: await command(root, "git", ["remote", "get-url", "origin"]),
+    commit: await command(root, "jj", ["--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"]),
+  }}, ...graph.resources.map((entry: {kind: "interface" | "package"; directory: string; source: Source}) => ({kind: entry.kind, source: entry.source,
+    directory: path.relative(workbench, path.resolve(workbench, entry.directory))}))];
+  let baseline: string | undefined;
+  try {
+    const host = JSON.parse(await fs.readFile("/etc/quixos/workspace-source.json", "utf8"));
+    if (await fs.realpath(host.workbenchRoot) === await fs.realpath(workbench)) baseline = JSON.parse(await fs.readFile(path.join(host.runtimeClosureRoot, "manifest.json"), "utf8")).workspacePlanPath;
+  } catch (error) {if (!["ENOENT", "EACCES"].includes((error as NodeJS.ErrnoException).code ?? "")) throw error;}
+  return {nodes, baseline};
+};
+
+/** Read-only source plan. Repositories are selected explicitly, including any
+ * parallel versions of the same resource; no guesses at a floating 'latest'. */
+export const planPinUpgrades = async (workbenchPath: string, spec: UpgradeSpec): Promise<UpgradePlan> => {
+  const workbench = await fs.realpath(workbenchPath);
+  if (!Array.isArray(spec.nodes) || !spec.nodes.length || spec.nodes.length > 100 || spec.nodes.filter((node) => node.kind === "workspace").length !== 1) throw new Error("Upgrade graph requires one workspace and at most 100 repositories");
+  if (spec.quixos) validSource(spec.quixos);
+  const keys = new Map<string, string>();
+  for (const node of spec.nodes) {
+    validSource(node.source);
+    if (!["workspace", "package", "interface"].includes(node.kind) || keys.has(sourceKey(node))) throw new Error("Duplicate/invalid upgrade resource identity");
+    keys.set(sourceKey(node), node.directory);
+  }
+  if (new Set(spec.nodes.map((node) => node.directory)).size !== spec.nodes.length) throw new Error("Upgrade directories must be distinct");
+  const nodes: NodePlan[] = [];
+  for (const node of spec.nodes) {
+    const root = await location(workbench, node.directory);
+    if (await command(root, "git", ["remote", "get-url", "origin"]) !== node.source.repository) throw new Error(`Upgrade origin differs from selected source: ${node.directory}`);
+    const loaded = await loadQuixosLock(path.join(root, "quixos.lock"));
+    if (!loaded.ok) throw new Error(`Invalid lock in ${node.directory}: ${loaded.diagnostics.map((entry) => entry.message).join("; ")}`);
+    const dependencies = loaded.lock.resources.map((resource) => keys.get(sourceKey(resource))).filter((value): value is string => Boolean(value));
+    nodes.push({...node, treeDigest: await treeDigest(root), dependencies: [...new Set(dependencies)], lockFiles: loaded.lock.sourceFiles ?? ["quixos.lock"]});
+  }
+  const ordered: NodePlan[] = [], remaining = [...nodes];
+  while (remaining.length) {
+    const index = remaining.findIndex((node) => node.dependencies.every((dependency) => ordered.some((entry) => entry.directory === dependency)));
+    if (index < 0) throw new Error("Cyclic source publication graph");
+    ordered.push(remaining.splice(index, 1)[0]);
+  }
+  const workspace = ordered.find((node) => node.kind === "workspace")!;
+  // Even unreferenced new resources are published before the root.
+  ordered.splice(ordered.indexOf(workspace), 1); ordered.push(workspace);
+  const plan = {schemaVersion: 1 as const, workbench, spec, nodes: ordered};
+  return {...plan, digest: contentDigest(plan)};
+};
+
+export type UpgradeEffects = {
+  check(node: NodePlan, root: string, output: string, spec: UpgradeSpec): Promise<void>;
+  snapshot(root: string): Promise<string>;
+  publish(root: string, commit: string): Promise<void>;
+};
+const effects: UpgradeEffects = {
+  async check(node, root, output, spec) {
+    if (node.kind === "workspace" && !spec.baseline && !spec.bootstrap) throw new Error("Upgrading a workspace requires its checked active baseline for major-review checks (or explicit bootstrap:true for a new workspace)");
+    // Publication checks consume already-published dependency revisions, never
+    // workbench dirty overlays masquerading as those immutable identities.
+    const snapshotMap = `${output}-published-dependencies.json`;
+    await fs.writeFile(snapshotMap, JSON.stringify({resources: []}), {flag: "wx"});
+    const result = node.kind === "workspace" ? await checkWorkspaceCandidate({root, output, snapshotMap, baseline: spec.baseline, reviews: spec.reviews})
+      : await checkResourceCandidate({root, output, kind: node.kind, source: node.source, snapshotMap});
+    if (result.blockers.length) throw new Error(`Refactor required in ${node.directory}: ${result.blockers.join("; ")}`);
+    const evolution = (result as {evolution?: {reviews: {accepted: boolean}[]}}).evolution;
+    if (evolution?.reviews.some((review) => !review.accepted)) throw new Error("Explicit semantic-major review required before publishing the workspace");
+  },
+  async snapshot(root) {
+    // Unlike checking, publication deliberately captures the working copy.
+    await command(root, "jj", ["status"]);
+    const conflicts = await command(root, "jj", ["resolve", "--list"]);
+    if (conflicts) throw new Error("Resolve source conflicts before publication");
+    const commit = await command(root, "jj", ["--ignore-working-copy", "log", "--no-graph", "-r", "@", "-T", "commit_id"]);
+    if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(commit)) throw new Error("Publication did not resolve an exact commit");
+    await command(root, "git", ["diff", "--exit-code", "--no-ext-diff", "--no-textconv", commit, "--"]);
+    const tracked = new Set((await command(root, "git", ["ls-tree", "-r", "--name-only", "-z", commit])).split("\0"));
+    for (const file of (await command(root, "git", ["ls-files", "--others", "--exclude-standard", "-z"])).split("\0").filter(Boolean)) if (!tracked.has(file)) throw new Error(`Uncaptured source file ${file}`);
+    return commit;
+  },
+  async publish(root, commit) {
+    const ref = `refs/tags/quixos-reachability/${commit}`;
+    const remote = await command(root, "git", ["ls-remote", "--refs", "origin", ref]);
+    if (remote && remote !== `${commit}\t${ref}`) throw new Error("Immutable publication ref conflict");
+    if (!remote) await command(root, "git", ["push", "origin", `${commit}:${ref}`]);
+    if (await command(root, "git", ["ls-remote", "--refs", "origin", ref]) !== `${commit}\t${ref}`) throw new Error("Publication response uncertain; retry the same journal");
+  },
+};
+
+/** Explicit --publish only. Append-only remote retention; never moves the
+ * workspace branch, activates code, or rolls back previously published nodes. */
+export const applyPinUpgrades = async (plan: UpgradePlan, journalId?: string, implementation: UpgradeEffects = effects, options: {acceptEdits?: boolean} = {}) => {
+  if (implementation === effects && !plan.spec.baseline && !plan.spec.bootstrap) throw new Error("Publication requires an active checked baseline or explicit bootstrap:true");
+  const {digest, ...body} = plan;
+  if (contentDigest(body) !== digest) throw new Error("Upgrade plan digest mismatch");
+  const directory = path.join(plan.workbench, ".quixos", "upgrades");
+  await fs.mkdir(directory, {recursive: true, mode: 0o700});
+  if (await fs.realpath(directory) !== directory) throw new Error("Upgrade journals must not cross symlinks");
+  const lock = await fs.open(path.join(directory, "writer.lock"), "wx", 0o600);
+  const id = journalId ?? randomUUID();
+  if (!/^[a-f0-9-]{36}$/.test(id)) {await lock.close(); await fs.unlink(path.join(directory, "writer.lock")); throw new Error("Invalid upgrade journal ID");}
+  const filename = path.join(directory, `${id}.json`);
+  try {
+    const journal: Journal = journalId ? JSON.parse(await fs.readFile(filename, "utf8")) : {schemaVersion: 1, plan, steps: []};
+    if (journal.plan.digest !== plan.digest) throw new Error("Upgrade journal belongs to another plan");
+    if (!journalId) await writeJournal(filename, journal);
+    for (const node of plan.nodes) {
+      const root = await location(plan.workbench, node.directory);
+      if (await command(root, "git", ["remote", "get-url", "origin"]) !== node.source.repository) throw new Error("Upgrade remote changed after planning");
+      let step = journal.steps.find((entry) => entry.directory === node.directory);
+      if (step?.phase === "published") continue;
+      if (!step) {
+        if (await treeDigest(root) !== node.treeDigest) throw new Error(`Stale upgrade plan: ${node.directory}`);
+        const files: StructuralRequest["files"] = [];
+        for (const file of node.lockFiles) {
+          const parsed = parseQuixosLockDocument(await fs.readFile(path.join(root, file), "utf8"));
+          if (!parsed.ok) throw new Error("Invalid lock during upgrade");
+          const edits = [];
+          for (const resource of parsed.document.resources) {
+            const dependency = plan.nodes.find((entry) => sourceKey(entry) === sourceKey(resource));
+            const published = dependency && journal.steps.find((entry) => entry.directory === dependency.directory && entry.phase === "published");
+            if (published?.commit && published.commit !== resource.source.commit) edits.push({operation: "dependency" as const, kind: resource.kind, name: resource.binding, source: {...resource.source, commit: published.commit}});
+          }
+          if (parsed.document.kind === "root" && plan.spec.quixos) edits.push({operation: "quixos-pin" as const, source: plan.spec.quixos});
+          if (edits.length) files.push({file, edits});
+        }
+        const structuralPlan = files.length ? await planStructure(root, {kind: node.kind, source: node.source, files}, process.env.QUIXOS_SNAPSHOT_MAP) : undefined;
+        step = {directory: node.directory, phase: "editing", treeDigest: node.treeDigest, structuralPlan, structuralJournal: structuralPlan ? randomUUID() : undefined};
+        journal.steps.push(step); await writeJournal(filename, journal);
+      }
+      if (step.phase === "editing") {
+        if (step.structuralPlan) await applyStructure(step.structuralPlan, step.structuralJournal);
+        else if (await treeDigest(root) !== step.treeDigest) throw new Error("Source changed before upgrade editing");
+        step.treeDigest = await treeDigest(root);
+        step.phase = "prepared";
+        await writeJournal(filename, journal);
+      }
+      if (step.phase === "refactor") {
+        const current = await treeDigest(root);
+        if (current !== step.treeDigest && !options.acceptEdits) throw new Error("Refactored source requires --accept-edits when resuming");
+        step.treeDigest = current; step.phase = "prepared"; await writeJournal(filename, journal);
+      }
+      if (await treeDigest(root) !== step.treeDigest) throw new Error(`Source changed during upgrade: ${node.directory}; inspect ${filename}`);
+      if (step.phase === "prepared") {
+        try {await implementation.check(node, root, path.join(directory, `${id}-${node.directory.replaceAll("/", "-")}-${randomUUID()}`), plan.spec);}
+        catch (error) {step.phase = "refactor"; await writeJournal(filename, journal); throw error;}
+        if (await treeDigest(root) !== step.treeDigest) throw new Error("Source changed while checking");
+        step.phase = "checked"; await writeJournal(filename, journal);
+      }
+      if (step.phase === "checked") {
+        step.commit = await implementation.snapshot(root);
+        if (await treeDigest(root) !== step.treeDigest) throw new Error("Publication snapshot changed checked files");
+        step.phase = "publishing"; await writeJournal(filename, journal);
+      }
+      await implementation.publish(root, step.commit!);
+      step.phase = "published"; await writeJournal(filename, journal);
+    }
+    // Keep subsequent automatic upgrades associated with the newly published
+    // identities, without renaming repositories or changing any selected branch.
+    // Explicit-spec callers without a managed graph retain the journal as their
+    // source of revisions instead.
+    const graphFile = path.join(plan.workbench, ".quixos/resource-graph.json");
+    let graphText: string | undefined;
+    try {graphText = await fs.readFile(graphFile, "utf8");} catch (error) {if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;}
+    if (graphText !== undefined) {
+      const previous = JSON.parse(graphText);
+      const snapshots = await Promise.all(previous.resources.map(async (entry: {kind: string; source: Source; directory: string}) => ({kind: entry.kind, ...entry.source, directory: await location(plan.workbench, path.relative(plan.workbench, path.resolve(plan.workbench, entry.directory)))})));
+      for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) {
+        const step = journal.steps.find((entry) => entry.directory === node.directory)!;
+        if (await treeDigest(await location(plan.workbench, node.directory)) !== step.treeDigest) throw new Error("Published source changed before workbench graph refresh");
+        snapshots.push({kind: node.kind, repository: node.source.repository, commit: step.commit, directory: path.join(plan.workbench, node.directory)});
+      }
+      const unique = [...new Map(snapshots.map((entry: {kind: string; repository: string; commit: string}) => [JSON.stringify([entry.kind, entry.repository, entry.commit]), entry])).values()];
+      const snapshotMap = path.join(directory, `${id}-published-snapshots.json`);
+      await fs.writeFile(snapshotMap, JSON.stringify({resources: unique}));
+      const resolveResource = await createGitCapabilityResolver({checkoutRoot: path.join(directory, `${id}-graph-resources`), snapshotMap, snapshotOnly: true});
+      const rootNode = plan.nodes.find((node) => node.kind === "workspace")!;
+      if (await treeDigest(await location(plan.workbench, rootNode.directory)) !== journal.steps.find((step) => step.directory === rootNode.directory)!.treeDigest) throw new Error("Root source changed before workbench graph refresh");
+      const compiled = await compileWorkspaceRepository({rootDirectory: await location(plan.workbench, rootNode.directory), resolveResource});
+      // Managed repositories need not currently be reachable from the workspace.
+      // Keep them discoverable/checkpointed until explicitly removed by the user.
+      const resources = new Map<string, ResolvedCapabilityResource>(compiled.resources.map((node) => [node.key, node]));
+      for (const node of plan.nodes.filter((entry) => entry.kind !== "workspace")) {
+        const step = journal.steps.find((entry) => entry.directory === node.directory)!;
+        const source = {resolver: "git" as const, repository: node.source.repository, commit: step.commit!};
+        const key = `${node.kind}\0${source.repository}\0${source.commit}`;
+        if (resources.has(key)) continue;
+        const directory = await location(plan.workbench, node.directory);
+        const standalone = await compileCapabilityResourceRepository({rootDirectory: directory, kind: node.kind as "package" | "interface", source, resolveResource});
+        for (const dependency of standalone.resources) resources.set(dependency.key, dependency);
+        resources.set(key, {key, kind: node.kind as "package" | "interface", source, directory, lock: standalone.lock, resource: standalone.resource, dependencies: standalone.directResources});
+      }
+      await writeJournal(graphFile, {formatVersion: 1, quixos: compiled.lock.quixos,
+        directResources: [...compiled.directResources.entries()].map(([bindingKey, node]) => {const [kind, binding] = bindingKey.split("\0"); return {kind, binding, resourceKey: node.key, directory: node.directory};}),
+        resources: [...resources.values()].map((node) => ({key: node.key, kind: node.kind, source: node.source, directory: node.directory,
+          resourceId: node.resource.kind === "interface" ? node.resource.revision.interfaceId : node.resource.revision.packageId,
+          revisionId: node.resource.revision.revisionId, dependencies: [...node.dependencies.entries()].map(([binding, dependency]) => ({binding, resourceKey: dependency.key}))})),
+      });
+    }
+    return {id, journal: filename, revisions: journal.steps.map((step) => ({directory: step.directory, commit: step.commit})), activated: false};
+  } catch (error) {throw new Error(`${error instanceof Error ? error.message : String(error)}; upgrade journal ${filename}`, {cause: error});}
+  finally {await lock.close(); await fs.unlink(path.join(directory, "writer.lock"));}
+};