From f352ac0c3b17ca5ba2c9de9818184c41d457b9ae Mon Sep 17 00:00:00 2001 From: "Timothy J. Aveni" Date: Sun, 20 Sep 2026 00:25:53 -0700 Subject: [PATCH] Build checked portable package artifacts and shared runtime assets --- PORTABLE_ARTIFACTS.md | 56 +++++++++ build-portable-artifacts.mjs | 213 +++++++++++++++++++++++++++++++++++ portable-artifacts.nix | 53 +++++++++ portable-service.mjs | 151 +++++++++++++++++++++++++ quixos-package-helpers.nix | 1 + 5 files changed, 474 insertions(+) create mode 100644 PORTABLE_ARTIFACTS.md create mode 100644 build-portable-artifacts.mjs create mode 100644 portable-artifacts.nix create mode 100644 portable-service.mjs diff --git a/PORTABLE_ARTIFACTS.md b/PORTABLE_ARTIFACTS.md new file mode 100644 index 0000000..cdff6aa --- /dev/null +++ b/PORTABLE_ARTIFACTS.md @@ -0,0 +1,56 @@ +# Checked portable artifacts + +`mkCaminoPortableArtifacts` is the fresh package builder. It regenerates candidate +bindings, typechecks the entry's dependency closure, bundles declared targets, +generates a process service when requested, and emits content manifests and +receipts. It has no migration step or package-defined shell hooks. + +```nix +# Inside a per-system flake output: +quixosPackages.checkedArtifacts = + { plan, generator, core, sharedRuntime, typescript, packageRevisionId }: + helpers.mkCaminoPortableArtifacts { + inherit pkgs plan core sharedRuntime typescript packageRevisionId; + protocol = generator; + src = ./.; + portable = { + entry = "src/portable.ts"; + bindingOutput = "src/gen/qx.ts"; + registryExport = "registry"; + targets = [ "browser" "server" ]; + }; + }; +``` + +The handler entry exports `registry = createRegistry(handlers)` from the generated +bindings. One registry contains many exports. The builder always supplies a server +implementation for the generated process service. A service-only package instead +omits `portable` and supplies `service.entry`, an ordinary JavaScript process entry +using the versioned stdin/stdout protocol. Custom services own their lifecycle and +may hold external resources. This first helper supports JavaScript custom service +entries; other languages can emit the same checked artifact contract independently. + +Additional portable source dependencies can be provided as a locked `nodeModules` directory. +The workspace supplies the exact SDK and compiler; packages cannot replace those +with their own versions. Authored source symlinks and non-bundled portable imports +other than the SDK and server Node builtins are rejected. This is build closure +checking, not a purity proof or security sandbox. + +The core/Automerge bundle and Replicache bundle are built once. Browser modules +import the core through a content-addressed gateway URL, independent of package +identity. Each manifest covers the shared bytes it uses. The gateway retains its +workspace authentication and explicit file registry; it never interprets an HTTP +path as a Nix store path. Applications cannot import Replicache through the SDK. + +`checked-artifacts.nix` evaluates a package's `checkedArtifacts` output without +activation. `loadCheckedArtifactWorld` then verifies manifest bytes, the trusted +generator identity, and exact regenerated binding sources. Receipts are consistency +evidence from a trusted Nix build, not cryptographic attestations of remote builds. +The separate manifest-only loader is useful for candidate diagnostics; activation +must use the checked loader. + +The root `camino-artifact-builds` check exercises actual reader/service outputs, +module and process execution, browser shared imports, rejection recovery, and +altered receipt rejection. No running workspace uses these outputs yet. Installing +the new world into the coordinator/replica host is LF-08/LF-11 integration work; +no adapter pretends this ABI works in the old server dispatcher. diff --git a/build-portable-artifacts.mjs b/build-portable-artifacts.mjs new file mode 100644 index 0000000..4c58c19 --- /dev/null +++ b/build-portable-artifacts.mjs @@ -0,0 +1,213 @@ +/** Trusted Nix build driver. No package-defined build hooks run between generation and witnessing. */ +import fs from "node:fs/promises"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { builtinModules } from "node:module"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; +const config = JSON.parse(await fs.readFile(process.argv[2], "utf8")); +const compiler = await import(pathToFileURL(config.compiler)); +const plan = JSON.parse(await fs.readFile(config.plan, "utf8")); +const pkg = plan.definition.packages.find((p) => p.revision === config.packageRevisionId); +if (!pkg || compiler.hexDigest(plan.definition) !== plan.digest) throw Error("Invalid checked execution plan/package"); +const output = process.env.out; +if (!output) throw Error("Missing Nix output"); +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const generatorDigest = digest(await fs.readFile(config.compiler)); +const runtime = config.portable + ? JSON.parse(await fs.readFile(path.join(config.sharedRuntime, "runtime.json"), "utf8")) + : { files: [] }; +const relative = (p) => compiler.artifactPath(p); +const implementations = []; +const generated = new Map(); +const recordFile = async (p, mediaType) => { + const bytes = await fs.readFile(path.join(output, relative(p))); + return { path: p, digest: digest(bytes), bytes: bytes.length, mediaType }; +}; +const copyRuntime = async () => { + await fs.mkdir(path.join(output, "share/quixos/shared"), { recursive: true }); + for (const file of runtime.files) { + relative(file.path); + const bytes = await fs.readFile(path.join(config.sharedRuntime, file.path)); + if (bytes.length !== file.bytes || digest(bytes) !== file.digest) throw Error("Shared runtime mismatch"); + await fs.writeFile(path.join(output, "share/quixos/shared", file.path), bytes); + } +}; +await copyRuntime(); +const sharedFiles = await Promise.all( + runtime.files.map((f) => recordFile(`share/quixos/shared/${f.path}`, f.mediaType)), +); +const coreFile = sharedFiles.find((f) => f.path.endsWith("/core.mjs")); +const coreURL = coreFile ? compiler.contentArtifactUrl(coreFile) : null; +const coreServer = path.join(config.sharedRuntime, "core.mjs"); +await fs.mkdir(path.join(output, "share/quixos/generated"), { recursive: true }); +const portable = config.portable; +if (portable && !/^[$A-Z_a-z][$\w]*$/.test(portable.registryExport)) + throw Error("Registry export must be a JavaScript identifier"); +const targets = portable ? [...new Set([...portable.targets, "server"])] : []; +for (const target of targets) { + if (!["browser", "server"].includes(target)) throw Error(`Unsupported JS build target ${target}`); + const work = path.resolve(`work-${target}`); + await fs.cp(config.source, work, { recursive: true, dereference: false }); + const writable = async (directory) => { + await fs.chmod(directory, 0o755); + for (const item of await fs.readdir(directory, { withFileTypes: true })) { + const file = path.join(directory, item.name); + if (item.isSymbolicLink()) throw Error("Authored package source cannot contain symlinks"); + if (item.isDirectory()) await writable(file); + else await fs.chmod(file, 0o644); + } + }; + await writable(work); + const modules = path.join(work, "node_modules"); + await fs.mkdir(path.join(modules, "@quixos"), { recursive: true }); + if (config.nodeModules) { + for (const item of await fs.readdir(config.nodeModules, { withFileTypes: true })) { + if (item.name === "@quixos") throw Error("Application dependencies cannot substitute the checked SDK"); + await fs.symlink(path.join(config.nodeModules, item.name), path.join(modules, item.name)); + } + } + await fs.symlink(config.core, path.join(modules, "@quixos/camino-replica-core")); + await fs.writeFile(path.join(work, "package.json"), JSON.stringify({ type: "module" })); + const bindingPath = relative(portable.bindingOutput); + const source = compiler.generatePortableTypeScriptBindings(plan.definition.bindingSchema, pkg.revision, { target }); + await fs.mkdir(path.dirname(path.join(work, bindingPath)), { recursive: true }); + await fs.writeFile(path.join(work, bindingPath), source); + generated.set(target, { file: path.join(work, bindingPath), source }); + await fs.writeFile(path.join(output, "share/quixos/generated", `${target}.ts`), source); + const entry = relative(portable.entry); + execFileSync( + config.node, + [ + config.tsc, + "--noEmit", + "--strict", + "--skipLibCheck", + "false", + "--target", + "es2024", + "--lib", + "es2024,dom,esnext.disposable", + "--module", + "nodenext", + "--moduleResolution", + "nodenext", + entry, + ], + { cwd: work, stdio: "inherit" }, + ); + const bundlePath = `share/quixos/portable/${target}/module.mjs`; + await fs.mkdir(path.dirname(path.join(output, bundlePath)), { recursive: true }); + const coreImport = target === "browser" ? coreURL : coreServer; + const metadata = path.join(work, "bundle-meta.json"); + execFileSync( + config.esbuild, + [ + entry, + "--bundle", + `--platform=${target === "browser" ? "browser" : "node"}`, + "--format=esm", + "--target=es2024", + `--outfile=${path.join(output, bundlePath)}`, + `--metafile=${metadata}`, + `--alias:@quixos/camino-replica-core=${coreImport}`, + `--external:${coreImport}`, + ], + { cwd: work, stdio: "inherit" }, + ); + const meta = JSON.parse(await fs.readFile(metadata, "utf8")); + const built = Object.values(meta.outputs).find((o) => o.entryPoint); + if (!built?.exports.includes(portable.registryExport)) throw Error("Bundle lacks the declared registry export"); + // Portable modules must contain their ordinary source dependencies and share only the checked SDK. + if ( + Object.values(meta.outputs).some((o) => + o.imports.some( + (i) => + i.external && + i.path !== coreImport && + !(target === "server" && builtinModules.includes(i.path.replace(/^node:/, ""))), + ), + ) + ) + throw Error("Unexpected external import in portable module"); + const exports = pkg.exports + .filter((e) => e.execution.kind !== "remote" && e.execution.targets.includes(target)) + .map((e) => e.id); + const body = { + id: `${target}-module`, + target, + compatibility: "quixos-js-module-v1", + path: bundlePath, + ports: "quixos-transaction-ports-v1", + registryExport: portable.registryExport, + exports, + files: [await recordFile(bundlePath, "text/javascript"), ...sharedFiles], + }; + implementations.push({ ...body, contentDigest: compiler.implementationDigest(body) }); +} +await fs.mkdir(path.join(output, "bin"), { recursive: true }); +await fs.mkdir(path.join(output, "share/quixos/service"), { recursive: true }); +let serviceEntry; +if (config.service.entry) { + if (!/\.m?js$/.test(config.service.entry)) + throw Error("Custom service entry must be JavaScript; use a separate typed/language build before packaging"); + serviceEntry = path.join(config.source, relative(config.service.entry)); +} else { + if (!portable) throw Error("Generated service requires a portable registry"); + if (pkg.exports.some((e) => e.execution.kind === "remote" || !e.execution.targets.includes("server"))) + throw Error("Generated service does not cover every package export; provide a custom service"); + serviceEntry = path.resolve("service-entry.mjs"); + await fs.writeFile( + serviceEntry, + `import { ${portable.registryExport} as registry } from ${JSON.stringify(path.join(output, "share/quixos/portable/server/module.mjs"))};\nimport {servePortableRegistry} from ${JSON.stringify(config.serviceAdapter)};\nservePortableRegistry(registry);\n`, + ); +} +const serviceModule = "share/quixos/service/main.mjs"; +execFileSync( + config.esbuild, + [ + serviceEntry, + "--bundle", + "--platform=node", + "--format=esm", + "--target=es2024", + `--external:${coreServer}`, + `--outfile=${path.join(output, serviceModule)}`, + ], + { stdio: "inherit" }, +); +await fs.writeFile( + path.join(output, "bin/service"), + `#!${config.shell}\nexec ${config.node} ${output}/${serviceModule} "$@"\n`, + { mode: 0o755 }, +); +const serviceBody = { + id: "service", + target: "server", + compatibility: "quixos-service-v1", + path: "bin/service", + ports: "quixos-transaction-ports-v1", + exports: pkg.exports.map((e) => e.id), + files: [ + await recordFile("bin/service", "text/x-shellscript"), + await recordFile(serviceModule, "text/javascript"), + ...sharedFiles, + ], +}; +implementations.unshift({ ...serviceBody, contentDigest: compiler.implementationDigest(serviceBody) }); +const manifest = compiler.parsePackageArtifacts({ + schemaVersion: 1, + packageRevisionId: pkg.revision, + bindingDigest: pkg.bindingDigest, + sourceDigest: pkg.sourceDigest, + versions: plan.definition.versions, + implementations, +}); +for (const { file, source } of generated.values()) + if ((await fs.readFile(file, "utf8")) !== source) throw Error("Generated bindings changed during build"); +const { receipt, sources } = compiler.expectedArtifactReceipt(plan, manifest, generatorDigest); +for (const [file, bytes] of sources) + if (!Buffer.from(bytes).equals(await fs.readFile(path.join(output, file)))) + throw Error("Receipt generation differs from compiled bindings"); +await fs.writeFile(path.join(output, "share/quixos/package-artifacts.json"), JSON.stringify(manifest, null, 2)); +await fs.writeFile(path.join(output, "share/quixos/build-receipt.json"), JSON.stringify(receipt, null, 2)); diff --git a/portable-artifacts.nix b/portable-artifacts.nix new file mode 100644 index 0000000..9d050e4 --- /dev/null +++ b/portable-artifacts.nix @@ -0,0 +1,53 @@ +# Fresh artifact builder. Candidate plans and runtime dependencies are supplied by the workspace compiler. +{ + pkgs, + src, + plan, + protocol, + core, + sharedRuntime, + typescript ? core.dependencies, + packageRevisionId, + portable ? null, + service ? { }, + nodeModules ? null, +}: +let + compiler = "${protocol}/libexec/quixos-protocol/execution-world.mjs"; + config = pkgs.writeText "camino-artifact-build.json" ( + builtins.toJSON { + inherit + plan + compiler + packageRevisionId + core + sharedRuntime + nodeModules + service + ; + source = src; + portable = + if portable == null then + null + else + { + inherit (portable) entry; + registryExport = portable.registryExport or "registry"; + bindingOutput = portable.bindingOutput or "src/gen/qx.ts"; + targets = + portable.targets or [ + "browser" + "server" + ]; + }; + node = "${pkgs.nodejs_24}/bin/node"; + tsc = "${typescript}/node_modules/typescript/bin/tsc"; + esbuild = "${pkgs.esbuild}/bin/esbuild"; + shell = "${pkgs.runtimeShell}"; + serviceAdapter = ./portable-service.mjs; + } + ); +in +pkgs.runCommand "camino-checked-artifacts" { } '' + ${pkgs.nodejs_24}/bin/node ${./build-portable-artifacts.mjs} ${config} +'' diff --git a/portable-service.mjs b/portable-service.mjs new file mode 100644 index 0000000..402a5e1 --- /dev/null +++ b/portable-service.mjs @@ -0,0 +1,151 @@ +/** Generated-service transport. The supervisor owns process isolation, deadlines and termination. + * stdout is reserved for this protocol; application diagnostics belong on stderr. */ +export function servePortableRegistry( + registry, + { maxBytes = 4 * 1024 * 1024, maxInvocations = 64, maxPortCalls = 10000 } = {}, +) { + const abi = "quixos-transaction-ports-v1"; + const active = new Map(); + const pending = new Map(); + let buffer = Buffer.alloc(0), + stopped = false; + const send = (message) => { + const encoded = JSON.stringify(message); + if (Buffer.byteLength(encoded) > maxBytes) throw Error("SERVICE_MESSAGE_LIMIT"); + // A slow or absent coordinator must not cause an unbounded stdout queue. + if (process.stdout.writableLength + Buffer.byteLength(encoded) > maxBytes * 2) throw Error("SERVICE_BACKPRESSURE"); + process.stdout.write(encoded + "\n"); + }; + const errorBody = (error) => ({ + code: typeof error?.code === "string" ? error.code.slice(0, 1024) : "PACKAGE_ERROR", + message: String(error?.message ?? error).slice(0, 4096), + }); + const cancel = (id, reason) => { + active.delete(id); + for (const [key, call] of pending) + if (call.invocation === id) { + pending.delete(key); + call.reject(reason); + } + }; + const fatal = (error) => { + if (stopped) return; + stopped = true; + for (const id of active.keys()) cancel(id, error); + process.stderr.write(String(error?.message ?? error) + "\n"); + process.exitCode = 1; + process.stdin.destroy(); + }; + const receive = (message) => { + if (!message || message.abi !== abi) throw Error("SERVICE_ABI_MISMATCH"); + if (message.kind === "shutdown") { + for (const id of active.keys()) cancel(id, Error("SERVICE_SHUTDOWN")); + stopped = true; + process.stdin.destroy(); + return; + } + if (message.kind === "cancel") { + cancel(message.invocation, Error("INVOCATION_CANCELLED")); + return; + } + if (message.kind === "port-result") { + const key = JSON.stringify([message.invocation, message.sequence]); + const call = pending.get(key); + if (!call && !active.has(message.invocation)) return; // A canceled call may already have a reply in flight. + if (!call || message.frame !== call.frame) throw Error("UNEXPECTED_PORT_RESULT"); + pending.delete(key); + if (message.status === "returned") call.resolve(message.output); + else if (message.status === "failed") + call.reject( + Object.assign(Error(String(message.error?.message ?? "Port failed")), { + code: String(message.error?.code ?? "PORT_FAILED"), + }), + ); + else throw Error("INVALID_PORT_RESULT"); + return; + } + if ( + message.kind !== "invoke" || + typeof message.invocation !== "string" || + typeof message.frame !== "string" || + !message.invocation || + !message.frame || + message.invocation.length > 4096 || + message.frame.length > 4096 + ) + throw Error("INVALID_INVOCATION"); + if (active.has(message.invocation) || active.size >= maxInvocations) throw Error("INVOCATION_LIMIT_OR_DUPLICATE"); + const token = {}; + active.set(message.invocation, token); + let sequence = 0; + const channel = (request) => { + if (active.get(message.invocation) !== token) return Promise.reject(Error("EXPIRED_INVOCATION")); + if (++sequence > maxPortCalls || pending.size >= maxPortCalls) return Promise.reject(Error("PORT_CALL_LIMIT")); + const key = JSON.stringify([message.invocation, sequence]); + return new Promise((resolve, reject) => { + pending.set(key, { invocation: message.invocation, frame: message.frame, resolve, reject }); + try { + send({ + abi, + kind: "port", + frame: message.frame, + invocation: message.invocation, + sequence, + portId: request.portId, + action: { ...request.action, ...(request.key === undefined ? {} : { key: request.key }) }, + input: request.input, + }); + } catch (error) { + pending.delete(key); + reject(error); + } + }); + }; + void registry + .invoke( + { exportId: message.exportId, workspace: message.workspace, receiver: message.receiver, input: message.input }, + channel, + ) + .then( + (output) => { + if (active.get(message.invocation) === token) + send({ abi, invocation: message.invocation, status: "returned", output }); + }, + (error) => { + if (active.get(message.invocation) === token) + send({ abi, invocation: message.invocation, status: "failed", error: errorBody(error) }); + }, + ) + .catch(fatal) + .finally(() => { + if (active.get(message.invocation) === token) cancel(message.invocation, Error("INVOCATION_ENDED")); + }); + }; + process.stdin.on("data", (bytes) => { + if (stopped) return; + try { + // Consume each line before retaining the remainder; combined transport chunks may contain many frames. + let offset = 0; + while (offset < bytes.length) { + const newline = bytes.indexOf(10, offset); + const end = newline < 0 ? bytes.length : newline; + if (buffer.length + end - offset > maxBytes) throw Error("SERVICE_MESSAGE_LIMIT"); + buffer = Buffer.concat([buffer, bytes.subarray(offset, end)]); + if (newline < 0) break; + receive(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffer))); + buffer = Buffer.alloc(0); + offset = newline + 1; + if (stopped) break; + } + } catch (error) { + fatal(error); + } + }); + process.stdin.on("end", () => { + if (buffer.length) fatal(Error("TRUNCATED_SERVICE_MESSAGE")); + for (const id of active.keys()) cancel(id, Error("COORDINATOR_DISCONNECTED")); + }); + process.stdin.on("error", fatal); + process.stdout.on("error", fatal); + send({ abi, kind: "ready", exports: registry.exports }); +} diff --git a/quixos-package-helpers.nix b/quixos-package-helpers.nix index c042e22..5f46f9d 100644 --- a/quixos-package-helpers.nix +++ b/quixos-package-helpers.nix @@ -846,6 +846,7 @@ let ); in { + mkCaminoPortableArtifacts = import ./portable-artifacts.nix; inherit mkQuixosPackageFlake mkTsPackageServer