diff --git a/README.md b/README.md index 7ad63eb..c71eedc 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Quixos protocol and capability compiler -This package owns the shared protobuf APIs for Camino, `quixos-orch`, package -runtimes, package descriptors, and generic runtime values. It also owns the v1 -capability authoring language and semantic compiler. +Shared protobuf APIs for Camino, orch, package runtimes/descriptors and values, +plus the capability language/compiler. Commands below are for backend development; +workspace authors use `qx-workspace check` for all verification. ## Capability language diff --git a/src/capability-language/authoring-check.ts b/src/capability-language/authoring-check.ts index 3dae757..023929e 100644 --- a/src/capability-language/authoring-check.ts +++ b/src/capability-language/authoring-check.ts @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import {appendFileSync} from "node:fs"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { authoringContext } from "./authoring-context.js"; @@ -21,13 +22,22 @@ export async function checkAuthoring(start: string, output: string, options: { b const report: { directory: string; checker: string; candidateOnly: true; activationEvidence: false; commit?: string; artifactPath?: string; blockers: string[]; phase: string; output: string; compilation?: "passed"; activationReadiness?: "preserve" | "migration-required" | "blocked"; migrationRequired?: string[] } = { directory, checker: checkerIdentity(), candidateOnly: true, activationEvidence: false, blockers: [], phase: "convergence", output, }; + const progress = async (running = true) => { + const file = path.join(output, "report.json"), temp = `${file}.tmp`; + await fs.writeFile(temp, JSON.stringify({...report, timings, running}, null, 2)); + await fs.rename(temp, file); + }; + await progress(); try { console.error(`[${new Date().toISOString()}] Check: capture source and converge dependencies`); // Serialize only source capture, not the potentially slow Nix build. // Repository-scoped agents can check separate immutable candidates in parallel. - const captured = await promisify(callback)("quixos-qx", ["converge", context.workbench, directory], { - maxBuffer: 4 * 1024 * 1024, env: {...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1"}, - }).catch(error => { + const capture = promisify(callback)("quixos-qx", ["converge", context.workbench, directory], { + maxBuffer: 4 * 1024 * 1024, env: {...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_TRACE_CAPTURE: "1"}, + }); + console.error(`Capture details: ${path.join(output, "capture.log")}`); + capture.child.stderr?.on("data", chunk => appendFileSync(path.join(output, "capture.log"), chunk)); + const captured = await capture.catch(error => { if (typeof error.stdout === "string" && error.stdout.trim().startsWith("{")) return {stdout: error.stdout}; throw error; }); @@ -40,6 +50,7 @@ export async function checkAuthoring(start: string, output: string, options: { b } report.commit = converged.candidate.commit; report.phase = "verification"; + await progress(); const buildStarted = performance.now(); console.error(`[${new Date().toISOString()}] Check: immutable Nix ${options.contractOnly ? "contract" : "verification"}; build output: ${path.join(output, "nix.log")}`); try { @@ -53,6 +64,7 @@ export async function checkAuthoring(start: string, output: string, options: { b report.compilation = "passed"; if (resource.kind === "workspace" && !options.contractOnly) { report.phase = "evolution"; + await progress(); let baseline = options.baseline; if (!baseline) { try { @@ -73,7 +85,7 @@ export async function checkAuthoring(start: string, output: string, options: { b } catch (error) { report.blockers.push(String(error instanceof Error ? error.message : error)); } timings.totalMs = Math.round(performance.now() - started); Object.assign(report, {timings}); - await fs.writeFile(path.join(output, "report.json"), JSON.stringify(report, null, 2)); + await progress(false); if (options.contractOnly) return report; const records = path.join(context.workbench, ".quixos/checks"); await fs.mkdir(records, { recursive: true }); diff --git a/src/capability-language/authoring-converge.ts b/src/capability-language/authoring-converge.ts index a34f7d2..200fd32 100644 --- a/src/capability-language/authoring-converge.ts +++ b/src/capability-language/authoring-converge.ts @@ -8,10 +8,17 @@ import { snapshotCommit } from "./checked-build.js"; import { loadQuixosLock, parseQuixosLockDocument, formatQuixosLockDocument, retentionTagForCommit, type GitSource } from "../resource-lock/index.js"; const execFile = promisify(callback); -const command = async (cwd: string, executable: string, args: string[]) => (await execFile(executable, args, { - cwd, maxBuffer: 4 * 1024 * 1024, - env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_SUBTREE_PUBLISH: "0", GIT_TERMINAL_PROMPT: "0" }, -})).stdout.trim(); +const command = async (cwd: string, executable: string, args: string[]) => { + const start = performance.now(); + // Do not log arguments: transports may contain credentials. Source identities + // remain in the normal checked result, not in this timing channel. + const label = `${path.basename(cwd)} ${executable} ${args[0]}`; + try { return (await execFile(executable, args, { + cwd, maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, QUIXOS_JJ_NO_CHECKPOINT: "1", QUIXOS_SUBTREE_PUBLISH: "0", GIT_TERMINAL_PROMPT: "0" }, + })).stdout.trim(); } + finally { if (process.env.QUIXOS_TRACE_CAPTURE === "1") console.error(`[${new Date().toISOString()}] Capture: ${label}: ${Math.round(performance.now() - start)}ms`); } +}; const identity = (kind: string, repository: string) => `${kind}\0${repository}`; export type AuthoringBlocker = { directory: string; phase: "resolution" | "dependency" | "source" | "publication" | "concurrent-edit"; message: string }; @@ -92,7 +99,9 @@ export async function convergeAuthoring(start: string, target = "root") { } finally { await rm(temporary, { force: true }); } } } + const snapshotStarted = performance.now(); const commit = await snapshotCommit(root); + if (process.env.QUIXOS_TRACE_CAPTURE === "1") console.error(`[${new Date().toISOString()}] Capture: ${directory} snapshot: ${Math.round(performance.now() - snapshotStarted)}ms`); phase = "publication"; const ref = retentionTagForCommit(commit); const remote = await command(root, "git", ["ls-remote", "--refs", node.source.repository, ref]); diff --git a/src/capability-language/scaffold-recipes.ts b/src/capability-language/scaffold-recipes.ts index 8f8e0e9..bbd0f2c 100644 --- a/src/capability-language/scaffold-recipes.ts +++ b/src/capability-language/scaffold-recipes.ts @@ -14,6 +14,8 @@ export type ScaffoldRecipe = { template?: "typescript" | "typescript-react"; source: Source; directory?: string; name?: string; id?: string; revision?: string; declaration?: string; + /** Initial authored files for a composed recipe; only used for a new package. */ + initialFiles?: Record; tools?: {quixos: Source; protocol: Source; helpers: Source; sdk: Source}; nixifyPluginUrl?: string; migration?: Omit & {contracts: Record}; @@ -69,7 +71,7 @@ export const scaffoldRecipe = async (root: string, command: "package" | "functio if (react) { create("src/component.tsx", `// Props are opaque at the platform boundary until capability generics exist.\nexport default function Component(_props: {camino: unknown; render: unknown; dispatch: (action: unknown) => void}) {\n return

${name}

Edit this component, then run qx-workspace check.

;\n}\n`); create("src/impl/sourceGet.ts", `import componentSource from "../component.js?browser-source";\nimport type {Implementation} from "../gen/qx.js";\nexport const handler: Implementation["sourceGet"] = () => componentSource;\n`); - create("src/browser-assets.d.ts", `declare module "*?browser-source" { const source: string; export default source; }\n`); + create("src/browser-assets.d.ts", `declare module "*?browser-source" { const source: string; export default source; }\ndeclare module "*.css" {}\n`); create("src/gen/web-studio-react-runtime.d.ts", reactPlatformTypes); } create(".gitignore", "node_modules/\ndist/\n.quixos/\nresult\n.yarn/install-state.gz\n"); @@ -161,5 +163,31 @@ export const scaffoldRecipe = async (root: string, command: "package" | "functio create("src/migrate.ts", `import {serveMigration} from "@quixos/camino-package-runtime";\n` + migrations.map((entry, index) => `import {handler as impl${index}} from ${JSON.stringify(`./${entry.file.slice(4, -3)}.js`)};\n`).join("") + `await serveMigration({${migrations.map((entry, index) => `${JSON.stringify(entry.id)}: impl${index}`).join(", ")}});\n`); } generated("descriptor.quixos-package.txtpb", `# Generated by qx-scaffold-v1\npackage_id: ${JSON.stringify(packageModel.id)}\npackage_revision_id: ${JSON.stringify(packageModel.revision)}\nruntime_protocol_version: "quixos-capabilities-v1"\n` + packageModel.exports.map((entry) => `exports: { export_id: ${JSON.stringify(entry.id)} runtime_symbol: ${JSON.stringify(entry.name)} }\n`).join("")); + if (spec.initialFiles) { + if (command !== "package") throw new Error("initialFiles is only valid when creating a package"); + const authored = spec.initialFiles["package.qx"]; + if (authored !== undefined) { + const parsed = parseQx(authored); + const declaration = [...walkSyntax(parsed.root)].find(n => n.kind === "packageResourceDecl"); + const literals = declaration?.children.filter(n => n.kind === "stringLiteral") ?? []; + const identifier = declaration?.children.find(n => n.kind === "identifier"); + if (parsed.diagnostics.length || literals.length !== 2 || !identifier || authored.slice(identifier.start, identifier.end) !== spec.name || + JSON.parse(authored.slice(literals[0].start, literals[0].end)) !== spec.id || JSON.parse(authored.slice(literals[1].start, literals[1].end)) !== spec.revision) + throw new Error("Initial package declaration must match the provisioned name, ID and revision"); + } + for (const [file, content] of Object.entries(spec.initialFiles)) { + if (typeof content !== "string" || ["quixos.lock", "flake.nix", "quixos.toolchain.json", "quixos.check.json", "package.json"].includes(file)) throw new Error(`Not an initial authored file: ${file}`); + const existing = files.findIndex(entry => entry.file === prefix + file); + if (existing >= 0) files.splice(existing, 1); + create(file, content); + } + // A composed React recipe supplies its own modules and complete server. + if (reactRecipe(spec)) { + for (const file of ["src/component.tsx", "src/impl/sourceGet.ts", "descriptor.quixos-package.txtpb"]) + if (!(file in spec.initialFiles)) {const index = files.findIndex(entry => entry.file === prefix + file); if (index >= 0) files.splice(index, 1);} + } + } return {kind: "package", source: spec.source, resourceRoot: spec.directory, validation: "syntax", files}; }; + +const reactRecipe = (spec: ScaffoldRecipe) => spec.template === "typescript-react" && spec.initialFiles?.["package.qx"] && spec.initialFiles?.["src/server.ts"]; diff --git a/src/capability-language/structural-plan.ts b/src/capability-language/structural-plan.ts index 2e8fbc8..d123309 100644 --- a/src/capability-language/structural-plan.ts +++ b/src/capability-language/structural-plan.ts @@ -22,7 +22,7 @@ export type StructuralRequest = { type Change = {file: string; before: string | null; after: string; mode: number}; type Journal = {schemaVersion: 1; id: string; root: string; phase: "prepared" | "complete"; changes: Change[]}; const safeFile = (file: string) => { - if (!/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*(?:\.gitignore|\.yarnrc.yml|[A-Za-z0-9_-][A-Za-z0-9_.-]*\.(?:qx|lock|ts|tsx|mjs|json|nix|txtpb))$/.test(file) + if (!/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*(?:\.gitignore|\.yarnrc.yml|[A-Za-z0-9_-][A-Za-z0-9_.-]*\.(?:qx|lock|ts|tsx|css|mjs|json|nix|txtpb))$/.test(file) || file.split("/").some((part) => [".git", ".jj", ".quixos", "node_modules"].includes(part))) throw new Error(`Unsafe scaffold path ${file}`); }; const read = async (root: string, file: string): Promise => { diff --git a/src/capability-language/tool-cli.ts b/src/capability-language/tool-cli.ts index e96fc20..7a6cae1 100644 --- a/src/capability-language/tool-cli.ts +++ b/src/capability-language/tool-cli.ts @@ -2,6 +2,7 @@ import { readFile, writeFile, mkdtemp, rm, realpath } from "node:fs/promises"; import { parseQx, formatQx, lintQx } from "./source.js"; import { scaffoldAtom } from "./scaffold.js"; +import {loadQxSources} from "./source-loader.js"; import { createGitCapabilityResolver } from "./git-resolver.js"; import { planEvolution } from "../capability-model/index.js"; import { snapshotRepository } from "./candidate-check.js"; @@ -41,6 +42,39 @@ const planSummary = (plan: Awaited>) => ({ const main = async () => { const [command, ...args] = process.argv.slice(2); + if (command === "scaffold-validate-qx") { + if (args.length !== 1) throw new Error("scaffold-validate-qx SOURCES_JSON"); + const sources = JSON.parse(args[0]); + if (!Array.isArray(sources) || sources.length > 100) throw new Error("Expected at most 100 scaffold sources"); + for (const entry of sources) { + if (!entry || typeof entry.file !== "string" || typeof entry.source !== "string" || entry.source.length > 1024*1024) throw new Error("Invalid scaffold source"); + const diagnostics = parseQx(entry.source, entry.file).diagnostics; + if (diagnostics.length) throw new Error(diagnostics.map(d => `${entry.file}:${d.line}:${d.column + 1}: ${d.message}`).join("\n")); + } + process.stdout.write('{"syntaxValid":true,"verificationEvidence":false}\n'); + return; + } + if (command === "scaffold-placement-binding") { + if (args.length !== 1) throw new Error("scaffold-placement-binding ROOT"); + const {source} = await loadQxSources(args[0]); + const syntax = parseQx(source); + const text = (n: {start: number; end: number}) => source.slice(n.start, n.end); + const matches: string[] = []; + for (const edge of walkSyntax(syntax.root)) { + if (edge.kind !== "edgeDecl") continue; + for (const endpoint of edge.children.filter(n => n.kind === "edgeEndpoint")) { + const target = endpoint.children.find(n => n.kind === "targetConstraint"); + if (!target || !syntax.tokens.some(t => t.start >= target.start && t.end <= target.end && t.kind === "INTERFACE") || + !target.children.some(n => n.kind === "identifier" && text(n) === "WebStudioPlaceable")) continue; + const edgeName = text(edge.children.find(n => n.kind === "identifier")!); + const projection = text(endpoint.children.find(n => n.kind === "identifier")!); + matches.push(`bind placements.resolve to edge ${edgeName}.${projection}.resolve;`); + } + } + if (matches.length !== 1) throw new Error(`Expected one canvas placement edge for WebStudioPlaceable; found ${matches.length}. Configure the workspace canvas before adding a bundle.`); + process.stdout.write(JSON.stringify({binding: matches[0]}) + "\n"); + return; + } if (command === "package-identity") { if (args.length !== 1) throw new Error("usage: quixos-qx package-identity PACKAGE_QX"); const authored = await readFile(args[0], "utf8"); @@ -91,12 +125,14 @@ const main = async () => { if (!args.length || args.length > 2) throw new Error("usage: quixos-qx converge WORKBENCH [REGISTERED_DIRECTORY] (join package writers first)"); const context = await authoringContext(args[0]); if (command === "converge") { + console.error(`[${new Date().toISOString()}] Capture: waiting for coordinator lock (120s limit)`); const result = spawnSync("flock", ["--exclusive", "--timeout", "120", "--conflict-exit-code", "75", path.join(context.workbench, ".quixos/converge.lock"), process.execPath, process.argv[1], "_converge", context.workbench, ...(args[1] ? [args[1]] : [])], {stdio: "inherit"}); if (result.error) throw result.error; if (result.status === 75) process.stderr.write("Timed out after 120 seconds waiting for source capture; inspect the active coordinator. No build lock is held.\n"); process.exitCode = result.status ?? 1; return; } + console.error(`[${new Date().toISOString()}] Capture: coordinator lock acquired`); const result = await convergeAuthoring(context.workbench, args[1]); process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); if (!result.converged) process.exitCode = 1; @@ -173,7 +209,7 @@ const main = async () => { const spec = await readSpec(specFile) as ScaffoldRecipe; if (!spec.name || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(spec.name) || !spec.id || !spec.revision || !spec.tools?.quixos) throw new Error("Interface scaffold requires name, id, revision and Quixos toolchain source"); const request: StructuralRequest = {kind: "interface", source: spec.source, files: [ - {file: "interface.qx", create: `interface ${spec.name} id ${JSON.stringify(spec.id)} revision ${JSON.stringify(spec.revision)} {\n}\n`}, + {file: "interface.qx", create: spec.declaration ?? `interface ${spec.name} id ${JSON.stringify(spec.id)} revision ${JSON.stringify(spec.revision)} {\n}\n`}, {file: "quixos.lock", create: formatQuixosLock({formatVersion: 1, quixos: {resolver: "git", ...spec.tools.quixos}, resources: []})}, {file: ".gitignore", create: ".quixos/\n"}, ]}; diff --git a/test/scaffold-recipes.test.ts b/test/scaffold-recipes.test.ts index 6b9605a..befd92f 100644 --- a/test/scaffold-recipes.test.ts +++ b/test/scaffold-recipes.test.ts @@ -22,6 +22,7 @@ test("React preset applies its browser build script and shared-platform imports" assert.match(await fs.readFile(path.join(root, "src/impl/sourceGet.ts"), "utf8"), /component.js\?browser-source/); assert.match(await fs.readFile(path.join(root, "flake.nix"), "utf8"), /browserSources = true/); assert.equal(await fs.readFile(path.join(root, "src/gen/web-studio-react-runtime.d.ts"), "utf8"), reactPlatformTypes); + assert.match(await fs.readFile(path.join(root, "src/browser-assets.d.ts"), "utf8"), /declare module "\*\.css"/); assert.equal(JSON.parse(await fs.readFile(path.join(root, "quixos.check.json"), "utf8")).options.messages["org.quixos.web-studio.ReactProps"].export, "opaqueReactPropsBinding"); await assert.rejects(fs.access(path.join(root, "quixos.scaffold.json"))); assert.equal(JSON.parse(await fs.readFile(path.join(root, "tsconfig.json"), "utf8")).compilerOptions.jsx, "react-jsx");