Build checked portable package artifacts and shared runtime assets

PORTABLE_ARTIFACTS.md

@@ -0,0 +1,56 @@
+# Checked portable artifacts
+
+`mkCaminoPortableArtifacts` is the fresh package builder. It regenerates candidate
+bindings, typechecks the entry's dependency closure, bundles declared targets,
+generates a process service when requested, and emits content manifests and
+receipts. It has no migration step or package-defined shell hooks.
+
+```nix
+# Inside a per-system flake output:
+quixosPackages.checkedArtifacts =
+  { plan, generator, core, sharedRuntime, typescript, packageRevisionId }:
+  helpers.mkCaminoPortableArtifacts {
+    inherit pkgs plan core sharedRuntime typescript packageRevisionId;
+    protocol = generator;
+    src = ./.;
+    portable = {
+      entry = "src/portable.ts";
+      bindingOutput = "src/gen/qx.ts";
+      registryExport = "registry";
+      targets = [ "browser" "server" ];
+    };
+  };
+```
+
+The handler entry exports `registry = createRegistry(handlers)` from the generated
+bindings. One registry contains many exports. The builder always supplies a server
+implementation for the generated process service. A service-only package instead
+omits `portable` and supplies `service.entry`, an ordinary JavaScript process entry
+using the versioned stdin/stdout protocol. Custom services own their lifecycle and
+may hold external resources. This first helper supports JavaScript custom service
+entries; other languages can emit the same checked artifact contract independently.
+
+Additional portable source dependencies can be provided as a locked `nodeModules` directory.
+The workspace supplies the exact SDK and compiler; packages cannot replace those
+with their own versions. Authored source symlinks and non-bundled portable imports
+other than the SDK and server Node builtins are rejected. This is build closure
+checking, not a purity proof or security sandbox.
+
+The core/Automerge bundle and Replicache bundle are built once. Browser modules
+import the core through a content-addressed gateway URL, independent of package
+identity. Each manifest covers the shared bytes it uses. The gateway retains its
+workspace authentication and explicit file registry; it never interprets an HTTP
+path as a Nix store path. Applications cannot import Replicache through the SDK.
+
+`checked-artifacts.nix` evaluates a package's `checkedArtifacts` output without
+activation. `loadCheckedArtifactWorld` then verifies manifest bytes, the trusted
+generator identity, and exact regenerated binding sources. Receipts are consistency
+evidence from a trusted Nix build, not cryptographic attestations of remote builds.
+The separate manifest-only loader is useful for candidate diagnostics; activation
+must use the checked loader.
+
+The root `camino-artifact-builds` check exercises actual reader/service outputs,
+module and process execution, browser shared imports, rejection recovery, and
+altered receipt rejection. No running workspace uses these outputs yet. Installing
+the new world into the coordinator/replica host is LF-08/LF-11 integration work;
+no adapter pretends this ABI works in the old server dispatcher.

build-portable-artifacts.mjs

@@ -0,0 +1,213 @@
+/** Trusted Nix build driver. No package-defined build hooks run between generation and witnessing. */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import { builtinModules } from "node:module";
+import { execFileSync } from "node:child_process";
+import { pathToFileURL } from "node:url";
+const config = JSON.parse(await fs.readFile(process.argv[2], "utf8"));
+const compiler = await import(pathToFileURL(config.compiler));
+const plan = JSON.parse(await fs.readFile(config.plan, "utf8"));
+const pkg = plan.definition.packages.find((p) => p.revision === config.packageRevisionId);
+if (!pkg || compiler.hexDigest(plan.definition) !== plan.digest) throw Error("Invalid checked execution plan/package");
+const output = process.env.out;
+if (!output) throw Error("Missing Nix output");
+const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
+const generatorDigest = digest(await fs.readFile(config.compiler));
+const runtime = config.portable
+  ? JSON.parse(await fs.readFile(path.join(config.sharedRuntime, "runtime.json"), "utf8"))
+  : { files: [] };
+const relative = (p) => compiler.artifactPath(p);
+const implementations = [];
+const generated = new Map();
+const recordFile = async (p, mediaType) => {
+  const bytes = await fs.readFile(path.join(output, relative(p)));
+  return { path: p, digest: digest(bytes), bytes: bytes.length, mediaType };
+};
+const copyRuntime = async () => {
+  await fs.mkdir(path.join(output, "share/quixos/shared"), { recursive: true });
+  for (const file of runtime.files) {
+    relative(file.path);
+    const bytes = await fs.readFile(path.join(config.sharedRuntime, file.path));
+    if (bytes.length !== file.bytes || digest(bytes) !== file.digest) throw Error("Shared runtime mismatch");
+    await fs.writeFile(path.join(output, "share/quixos/shared", file.path), bytes);
+  }
+};
+await copyRuntime();
+const sharedFiles = await Promise.all(
+  runtime.files.map((f) => recordFile(`share/quixos/shared/${f.path}`, f.mediaType)),
+);
+const coreFile = sharedFiles.find((f) => f.path.endsWith("/core.mjs"));
+const coreURL = coreFile ? compiler.contentArtifactUrl(coreFile) : null;
+const coreServer = path.join(config.sharedRuntime, "core.mjs");
+await fs.mkdir(path.join(output, "share/quixos/generated"), { recursive: true });
+const portable = config.portable;
+if (portable && !/^[$A-Z_a-z][$\w]*$/.test(portable.registryExport))
+  throw Error("Registry export must be a JavaScript identifier");
+const targets = portable ? [...new Set([...portable.targets, "server"])] : [];
+for (const target of targets) {
+  if (!["browser", "server"].includes(target)) throw Error(`Unsupported JS build target ${target}`);
+  const work = path.resolve(`work-${target}`);
+  await fs.cp(config.source, work, { recursive: true, dereference: false });
+  const writable = async (directory) => {
+    await fs.chmod(directory, 0o755);
+    for (const item of await fs.readdir(directory, { withFileTypes: true })) {
+      const file = path.join(directory, item.name);
+      if (item.isSymbolicLink()) throw Error("Authored package source cannot contain symlinks");
+      if (item.isDirectory()) await writable(file);
+      else await fs.chmod(file, 0o644);
+    }
+  };
+  await writable(work);
+  const modules = path.join(work, "node_modules");
+  await fs.mkdir(path.join(modules, "@quixos"), { recursive: true });
+  if (config.nodeModules) {
+    for (const item of await fs.readdir(config.nodeModules, { withFileTypes: true })) {
+      if (item.name === "@quixos") throw Error("Application dependencies cannot substitute the checked SDK");
+      await fs.symlink(path.join(config.nodeModules, item.name), path.join(modules, item.name));
+    }
+  }
+  await fs.symlink(config.core, path.join(modules, "@quixos/camino-replica-core"));
+  await fs.writeFile(path.join(work, "package.json"), JSON.stringify({ type: "module" }));
+  const bindingPath = relative(portable.bindingOutput);
+  const source = compiler.generatePortableTypeScriptBindings(plan.definition.bindingSchema, pkg.revision, { target });
+  await fs.mkdir(path.dirname(path.join(work, bindingPath)), { recursive: true });
+  await fs.writeFile(path.join(work, bindingPath), source);
+  generated.set(target, { file: path.join(work, bindingPath), source });
+  await fs.writeFile(path.join(output, "share/quixos/generated", `${target}.ts`), source);
+  const entry = relative(portable.entry);
+  execFileSync(
+    config.node,
+    [
+      config.tsc,
+      "--noEmit",
+      "--strict",
+      "--skipLibCheck",
+      "false",
+      "--target",
+      "es2024",
+      "--lib",
+      "es2024,dom,esnext.disposable",
+      "--module",
+      "nodenext",
+      "--moduleResolution",
+      "nodenext",
+      entry,
+    ],
+    { cwd: work, stdio: "inherit" },
+  );
+  const bundlePath = `share/quixos/portable/${target}/module.mjs`;
+  await fs.mkdir(path.dirname(path.join(output, bundlePath)), { recursive: true });
+  const coreImport = target === "browser" ? coreURL : coreServer;
+  const metadata = path.join(work, "bundle-meta.json");
+  execFileSync(
+    config.esbuild,
+    [
+      entry,
+      "--bundle",
+      `--platform=${target === "browser" ? "browser" : "node"}`,
+      "--format=esm",
+      "--target=es2024",
+      `--outfile=${path.join(output, bundlePath)}`,
+      `--metafile=${metadata}`,
+      `--alias:@quixos/camino-replica-core=${coreImport}`,
+      `--external:${coreImport}`,
+    ],
+    { cwd: work, stdio: "inherit" },
+  );
+  const meta = JSON.parse(await fs.readFile(metadata, "utf8"));
+  const built = Object.values(meta.outputs).find((o) => o.entryPoint);
+  if (!built?.exports.includes(portable.registryExport)) throw Error("Bundle lacks the declared registry export");
+  // Portable modules must contain their ordinary source dependencies and share only the checked SDK.
+  if (
+    Object.values(meta.outputs).some((o) =>
+      o.imports.some(
+        (i) =>
+          i.external &&
+          i.path !== coreImport &&
+          !(target === "server" && builtinModules.includes(i.path.replace(/^node:/, ""))),
+      ),
+    )
+  )
+    throw Error("Unexpected external import in portable module");
+  const exports = pkg.exports
+    .filter((e) => e.execution.kind !== "remote" && e.execution.targets.includes(target))
+    .map((e) => e.id);
+  const body = {
+    id: `${target}-module`,
+    target,
+    compatibility: "quixos-js-module-v1",
+    path: bundlePath,
+    ports: "quixos-transaction-ports-v1",
+    registryExport: portable.registryExport,
+    exports,
+    files: [await recordFile(bundlePath, "text/javascript"), ...sharedFiles],
+  };
+  implementations.push({ ...body, contentDigest: compiler.implementationDigest(body) });
+}
+await fs.mkdir(path.join(output, "bin"), { recursive: true });
+await fs.mkdir(path.join(output, "share/quixos/service"), { recursive: true });
+let serviceEntry;
+if (config.service.entry) {
+  if (!/\.m?js$/.test(config.service.entry))
+    throw Error("Custom service entry must be JavaScript; use a separate typed/language build before packaging");
+  serviceEntry = path.join(config.source, relative(config.service.entry));
+} else {
+  if (!portable) throw Error("Generated service requires a portable registry");
+  if (pkg.exports.some((e) => e.execution.kind === "remote" || !e.execution.targets.includes("server")))
+    throw Error("Generated service does not cover every package export; provide a custom service");
+  serviceEntry = path.resolve("service-entry.mjs");
+  await fs.writeFile(
+    serviceEntry,
+    `import { ${portable.registryExport} as registry } from ${JSON.stringify(path.join(output, "share/quixos/portable/server/module.mjs"))};\nimport {servePortableRegistry} from ${JSON.stringify(config.serviceAdapter)};\nservePortableRegistry(registry);\n`,
+  );
+}
+const serviceModule = "share/quixos/service/main.mjs";
+execFileSync(
+  config.esbuild,
+  [
+    serviceEntry,
+    "--bundle",
+    "--platform=node",
+    "--format=esm",
+    "--target=es2024",
+    `--external:${coreServer}`,
+    `--outfile=${path.join(output, serviceModule)}`,
+  ],
+  { stdio: "inherit" },
+);
+await fs.writeFile(
+  path.join(output, "bin/service"),
+  `#!${config.shell}\nexec ${config.node} ${output}/${serviceModule} "$@"\n`,
+  { mode: 0o755 },
+);
+const serviceBody = {
+  id: "service",
+  target: "server",
+  compatibility: "quixos-service-v1",
+  path: "bin/service",
+  ports: "quixos-transaction-ports-v1",
+  exports: pkg.exports.map((e) => e.id),
+  files: [
+    await recordFile("bin/service", "text/x-shellscript"),
+    await recordFile(serviceModule, "text/javascript"),
+    ...sharedFiles,
+  ],
+};
+implementations.unshift({ ...serviceBody, contentDigest: compiler.implementationDigest(serviceBody) });
+const manifest = compiler.parsePackageArtifacts({
+  schemaVersion: 1,
+  packageRevisionId: pkg.revision,
+  bindingDigest: pkg.bindingDigest,
+  sourceDigest: pkg.sourceDigest,
+  versions: plan.definition.versions,
+  implementations,
+});
+for (const { file, source } of generated.values())
+  if ((await fs.readFile(file, "utf8")) !== source) throw Error("Generated bindings changed during build");
+const { receipt, sources } = compiler.expectedArtifactReceipt(plan, manifest, generatorDigest);
+for (const [file, bytes] of sources)
+  if (!Buffer.from(bytes).equals(await fs.readFile(path.join(output, file))))
+    throw Error("Receipt generation differs from compiled bindings");
+await fs.writeFile(path.join(output, "share/quixos/package-artifacts.json"), JSON.stringify(manifest, null, 2));
+await fs.writeFile(path.join(output, "share/quixos/build-receipt.json"), JSON.stringify(receipt, null, 2));

portable-artifacts.nix

@@ -0,0 +1,53 @@
+# Fresh artifact builder. Candidate plans and runtime dependencies are supplied by the workspace compiler.
+{
+  pkgs,
+  src,
+  plan,
+  protocol,
+  core,
+  sharedRuntime,
+  typescript ? core.dependencies,
+  packageRevisionId,
+  portable ? null,
+  service ? { },
+  nodeModules ? null,
+}:
+let
+  compiler = "${protocol}/libexec/quixos-protocol/execution-world.mjs";
+  config = pkgs.writeText "camino-artifact-build.json" (
+    builtins.toJSON {
+      inherit
+        plan
+        compiler
+        packageRevisionId
+        core
+        sharedRuntime
+        nodeModules
+        service
+        ;
+      source = src;
+      portable =
+        if portable == null then
+          null
+        else
+          {
+            inherit (portable) entry;
+            registryExport = portable.registryExport or "registry";
+            bindingOutput = portable.bindingOutput or "src/gen/qx.ts";
+            targets =
+              portable.targets or [
+                "browser"
+                "server"
+              ];
+          };
+      node = "${pkgs.nodejs_24}/bin/node";
+      tsc = "${typescript}/node_modules/typescript/bin/tsc";
+      esbuild = "${pkgs.esbuild}/bin/esbuild";
+      shell = "${pkgs.runtimeShell}";
+      serviceAdapter = ./portable-service.mjs;
+    }
+  );
+in
+pkgs.runCommand "camino-checked-artifacts" { } ''
+  ${pkgs.nodejs_24}/bin/node ${./build-portable-artifacts.mjs} ${config}
+''

portable-service.mjs

@@ -0,0 +1,151 @@
+/** Generated-service transport. The supervisor owns process isolation, deadlines and termination.
+ * stdout is reserved for this protocol; application diagnostics belong on stderr. */
+export function servePortableRegistry(
+  registry,
+  { maxBytes = 4 * 1024 * 1024, maxInvocations = 64, maxPortCalls = 10000 } = {},
+) {
+  const abi = "quixos-transaction-ports-v1";
+  const active = new Map();
+  const pending = new Map();
+  let buffer = Buffer.alloc(0),
+    stopped = false;
+  const send = (message) => {
+    const encoded = JSON.stringify(message);
+    if (Buffer.byteLength(encoded) > maxBytes) throw Error("SERVICE_MESSAGE_LIMIT");
+    // A slow or absent coordinator must not cause an unbounded stdout queue.
+    if (process.stdout.writableLength + Buffer.byteLength(encoded) > maxBytes * 2) throw Error("SERVICE_BACKPRESSURE");
+    process.stdout.write(encoded + "\n");
+  };
+  const errorBody = (error) => ({
+    code: typeof error?.code === "string" ? error.code.slice(0, 1024) : "PACKAGE_ERROR",
+    message: String(error?.message ?? error).slice(0, 4096),
+  });
+  const cancel = (id, reason) => {
+    active.delete(id);
+    for (const [key, call] of pending)
+      if (call.invocation === id) {
+        pending.delete(key);
+        call.reject(reason);
+      }
+  };
+  const fatal = (error) => {
+    if (stopped) return;
+    stopped = true;
+    for (const id of active.keys()) cancel(id, error);
+    process.stderr.write(String(error?.message ?? error) + "\n");
+    process.exitCode = 1;
+    process.stdin.destroy();
+  };
+  const receive = (message) => {
+    if (!message || message.abi !== abi) throw Error("SERVICE_ABI_MISMATCH");
+    if (message.kind === "shutdown") {
+      for (const id of active.keys()) cancel(id, Error("SERVICE_SHUTDOWN"));
+      stopped = true;
+      process.stdin.destroy();
+      return;
+    }
+    if (message.kind === "cancel") {
+      cancel(message.invocation, Error("INVOCATION_CANCELLED"));
+      return;
+    }
+    if (message.kind === "port-result") {
+      const key = JSON.stringify([message.invocation, message.sequence]);
+      const call = pending.get(key);
+      if (!call && !active.has(message.invocation)) return; // A canceled call may already have a reply in flight.
+      if (!call || message.frame !== call.frame) throw Error("UNEXPECTED_PORT_RESULT");
+      pending.delete(key);
+      if (message.status === "returned") call.resolve(message.output);
+      else if (message.status === "failed")
+        call.reject(
+          Object.assign(Error(String(message.error?.message ?? "Port failed")), {
+            code: String(message.error?.code ?? "PORT_FAILED"),
+          }),
+        );
+      else throw Error("INVALID_PORT_RESULT");
+      return;
+    }
+    if (
+      message.kind !== "invoke" ||
+      typeof message.invocation !== "string" ||
+      typeof message.frame !== "string" ||
+      !message.invocation ||
+      !message.frame ||
+      message.invocation.length > 4096 ||
+      message.frame.length > 4096
+    )
+      throw Error("INVALID_INVOCATION");
+    if (active.has(message.invocation) || active.size >= maxInvocations) throw Error("INVOCATION_LIMIT_OR_DUPLICATE");
+    const token = {};
+    active.set(message.invocation, token);
+    let sequence = 0;
+    const channel = (request) => {
+      if (active.get(message.invocation) !== token) return Promise.reject(Error("EXPIRED_INVOCATION"));
+      if (++sequence > maxPortCalls || pending.size >= maxPortCalls) return Promise.reject(Error("PORT_CALL_LIMIT"));
+      const key = JSON.stringify([message.invocation, sequence]);
+      return new Promise((resolve, reject) => {
+        pending.set(key, { invocation: message.invocation, frame: message.frame, resolve, reject });
+        try {
+          send({
+            abi,
+            kind: "port",
+            frame: message.frame,
+            invocation: message.invocation,
+            sequence,
+            portId: request.portId,
+            action: { ...request.action, ...(request.key === undefined ? {} : { key: request.key }) },
+            input: request.input,
+          });
+        } catch (error) {
+          pending.delete(key);
+          reject(error);
+        }
+      });
+    };
+    void registry
+      .invoke(
+        { exportId: message.exportId, workspace: message.workspace, receiver: message.receiver, input: message.input },
+        channel,
+      )
+      .then(
+        (output) => {
+          if (active.get(message.invocation) === token)
+            send({ abi, invocation: message.invocation, status: "returned", output });
+        },
+        (error) => {
+          if (active.get(message.invocation) === token)
+            send({ abi, invocation: message.invocation, status: "failed", error: errorBody(error) });
+        },
+      )
+      .catch(fatal)
+      .finally(() => {
+        if (active.get(message.invocation) === token) cancel(message.invocation, Error("INVOCATION_ENDED"));
+      });
+  };
+  process.stdin.on("data", (bytes) => {
+    if (stopped) return;
+    try {
+      // Consume each line before retaining the remainder; combined transport chunks may contain many frames.
+      let offset = 0;
+      while (offset < bytes.length) {
+        const newline = bytes.indexOf(10, offset);
+        const end = newline < 0 ? bytes.length : newline;
+        if (buffer.length + end - offset > maxBytes) throw Error("SERVICE_MESSAGE_LIMIT");
+        buffer = Buffer.concat([buffer, bytes.subarray(offset, end)]);
+        if (newline < 0) break;
+        receive(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffer)));
+        buffer = Buffer.alloc(0);
+        offset = newline + 1;
+        if (stopped) break;
+      }
+    } catch (error) {
+      fatal(error);
+    }
+  });
+  process.stdin.on("end", () => {
+    if (buffer.length) fatal(Error("TRUNCATED_SERVICE_MESSAGE"));
+    for (const id of active.keys()) cancel(id, Error("COORDINATOR_DISCONNECTED"));
+  });
+  process.stdin.on("error", fatal);
+  process.stdout.on("error", fatal);
+  send({ abi, kind: "ready", exports: registry.exports });
+}

quixos-package-helpers.nix

@@ -846,6 +846,7 @@ let
     );
 in
 {
+  mkCaminoPortableArtifacts = import ./portable-artifacts.nix;
   inherit
     mkQuixosPackageFlake
     mkTsPackageServer