Use a combined cache derivation by default

src/FetchCommand.ts

@@ -0,0 +1,58 @@
+import { Command, Option } from "clipanion";
+
+import {
+  Cache,
+  CommandContext,
+  Configuration,
+  Project,
+  StreamReport,
+  structUtils,
+} from "@yarnpkg/core";
+
+// Internal command that performs the fetch step.
+// Used from within Nix to build the cache for the project.
+export default class FetchCommand extends Command<CommandContext> {
+  static paths = [[`nixify`, `fetch`]];
+  locator = Option.String({ required: false });
+
+  async execute() {
+    const configuration = await Configuration.find(
+      this.context.cwd,
+      this.context.plugins,
+    );
+    const { project } = await Project.find(configuration, this.context.cwd);
+    const cache = await Cache.find(configuration);
+
+    const fetcher = configuration.makeFetcher();
+    const report = await StreamReport.start(
+      { configuration, stdout: this.context.stdout },
+      async (report) => {
+        if (this.locator) {
+          const { locatorHash } = structUtils.parseLocator(this.locator, true);
+          const pkg = project.originalPackages.get(locatorHash);
+          if (!pkg) {
+            report.reportError(0, `Invalid locator: ${this.locator}`);
+            return;
+          }
+
+          await fetcher.fetch(pkg, {
+            checksums: project.storedChecksums,
+            project,
+            cache,
+            fetcher,
+            report,
+          });
+        } else {
+          await report.startTimerPromise(`Resolution step`, async () => {
+            await project.resolveEverything({ report, lockfileOnly: true });
+          });
+          await report.startTimerPromise(`Fetch step`, async () => {
+            await project.fetchEverything({ cache, report, fetcher });
+          });
+        }
+      },
+    );
+
+    return report.exitCode();
+  }
+}

src/FetchOneCommand.ts

@@ -1,50 +0,0 @@
-import { Command, Option } from "clipanion";
-
-import {
-  Cache,
-  CommandContext,
-  Configuration,
-  Project,
-  StreamReport,
-  structUtils,
-} from "@yarnpkg/core";
-
-// Internal command that fetches a single locator.
-// Used from within Nix to build the cache for the project.
-export default class FetchOneCommand extends Command<CommandContext> {
-  static paths = [[`nixify`, `fetch-one`]];
-  locator = Option.String();
-
-  async execute() {
-    const configuration = await Configuration.find(
-      this.context.cwd,
-      this.context.plugins,
-    );
-    const { project } = await Project.find(configuration, this.context.cwd);
-    const cache = await Cache.find(configuration);
-
-    const fetcher = configuration.makeFetcher();
-
-    const report = await StreamReport.start(
-      { configuration, stdout: this.context.stdout },
-      async (report) => {
-        const { locatorHash } = structUtils.parseLocator(this.locator, true);
-        const pkg = project.originalPackages.get(locatorHash);
-        if (!pkg) {
-          report.reportError(0, `Invalid locator: ${this.locator}`);
-          return;
-        }
-
-        await fetcher.fetch(pkg, {
-          checksums: project.storedChecksums,
-          project,
-          cache,
-          fetcher,
-          report,
-        });
-      },
-    );
-
-    return report.exitCode();
-  }
-}

src/generate.ts

@@ -20,8 +20,12 @@ import defaultExprTmpl from "./tmpl/default.nix.in";
 import projectExprTmpl from "./tmpl/yarn-project.nix.in";
 import {
   computeFixedOutputStorePath,
+  hexToSri,
   sanitizeDerivationName,
+  sriToHex,
 } from "./nixUtils";
+import { writeNarStream, writeNarStrings } from "./narUtils";
+import { createHash } from "crypto";
 
 const isYarn3 = YarnVersion?.startsWith("3.") || false;
 
@@ -48,6 +52,13 @@ export default async (
     return;
   }
 
+  // Config validation.
+  const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
+  const individualDrvs = configuration.get(`individualNixPackaging`);
+  if (isolatedBuilds.length > 0 && !individualDrvs) {
+    throw Error(`isolatedNixBuilds requires individualNixPackaging to be set`);
+  }
+
   // Determine relative paths for Nix path literals.
   const nixExprPath = configuration.get(`nixExprPath`);
 
@@ -55,14 +66,13 @@ export default async (
   let yarnBinExpr: string;
   if (yarnPathAbs === null) {
     // Assume the current running script is the correct Yarn.
-    const hexHash = await hashUtils.checksumFile(
+    const sha512 = await hashUtils.checksumFile(
       process.argv[1] as PortablePath,
     );
-    const sriHash = "sha512-" + Buffer.from(hexHash, "hex").toString("base64");
     yarnBinExpr = [
       "fetchurl {",
       `  url = "https://repo.yarnpkg.com/${YarnVersion!}/packages/yarnpkg-cli/bin/yarn.js";`,
-      `  hash = "${sriHash}";`,
+      `  hash = "${hexToSri(sha512)}";`,
       "}",
     ].join("\n  ");
   } else if (yarnPathAbs.startsWith(cwd)) {
@@ -113,16 +123,14 @@ export default async (
       ppath.resolve(cwd, "yarn.lock" as PortablePath),
     );
 
-  // Build a list of cache entries so Nix can fetch them.
-  // TODO: See if we can use Nix fetchurl for npm: dependencies.
-  interface CacheEntry {
-    originalFilename: Filename;
-    filename: Filename;
-    sha512: string;
+  // Collect all the cache files used.
+  interface CacheFile {
+    pkg: Package;
+    checksum: string | undefined;
+    cachePath: PortablePath;
   }
-  const cacheEntries: Map<string, CacheEntry> = new Map();
-
-  const cacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
+  const cacheFiles = new Map<Filename, CacheFile>();
+  const allCacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
   const cacheOptions = { unstablePackages: project.conditionalLocators };
   for (const pkg of project.storedPackages.values()) {
     const { locatorHash } = pkg;
@@ -132,36 +140,83 @@ export default async (
       : cache.getLocatorPath(pkg, checksum || null);
     if (!cachePath) continue;
 
-    const filename = ppath.basename(cachePath);
-    if (!cacheFiles.has(filename)) continue;
+    if (!allCacheFiles.has(ppath.basename(cachePath))) continue;
 
-    const locatorStr = structUtils.stringifyLocator(pkg);
-    const sha512 = checksum
-      ? checksum.split(`/`).pop()!
-      : await hashUtils.checksumFile(cachePath);
-    cacheEntries.set(locatorStr, {
-      originalFilename: filename,
-      // Rebuild the filename, because the cache file we're operating on may be
-      // from the mirror directory, which uses different naming.
-      filename: checksum
-        ? cache.getChecksumFilename(pkg, checksum)
-        : cache.getVersionFilename(pkg),
-      sha512,
-    });
+    // Rebuild the filename, because the cache file we're operating on may be
+    // from the mirror directory, which uses different naming.
+    const filename = checksum
+      ? cache.getChecksumFilename(pkg, checksum)
+      : cache.getVersionFilename(pkg);
+
+    cacheFiles.set(filename, { pkg, checksum, cachePath });
   }
 
-  let cacheEntriesCode = `cacheEntries = {\n`;
-  for (const locatorStr of [...cacheEntries.keys()].sort()) {
-    const entry = cacheEntries.get(locatorStr)!;
-    cacheEntriesCode += `${json(locatorStr)} = { ${[
-      `filename = ${json(entry.filename)};`,
-      `sha512 = ${json(entry.sha512)};`,
-    ].join(` `)} };\n`;
+  interface CacheEntry {
+    cachePath: PortablePath;
+    filename: Filename;
+    hash: string;
+  }
+  const cacheEntries = new Map<string, CacheEntry>();
+  let cacheEntriesCode = "";
+  let combinedHash = "";
+  if (individualDrvs) {
+    // Build a list of cache entries so Nix can fetch them.
+    // TODO: See if we can use Nix fetchurl for npm: dependencies.
+    for (const [
+      filename,
+      { pkg, checksum, cachePath },
+    ] of cacheFiles.entries()) {
+      const locatorStr = structUtils.stringifyLocator(pkg);
+      const sha512 = checksum
+        ? checksum.split(`/`).pop()!
+        : await hashUtils.checksumFile(cachePath);
+      cacheEntries.set(locatorStr, {
+        cachePath,
+        filename,
+        hash: hexToSri(sha512),
+      });
+    }
+
+    cacheEntriesCode = `cacheEntries = {\n`;
+    for (const locatorStr of [...cacheEntries.keys()].sort()) {
+      const entry = cacheEntries.get(locatorStr)!;
+      cacheEntriesCode += `${json(locatorStr)} = { ${[
+        `filename = ${json(entry.filename)};`,
+        `hash = "${entry.hash}";`,
+      ].join(` `)} };\n`;
+    }
+    cacheEntriesCode += `};`;
+  } else {
+    // Hash a NAR of just the cache files we use.
+    const hasher = createHash("sha512");
+    writeNarStrings(hasher, "nix-archive-1", "(", "type", "directory");
+    for (const filename of [...cacheFiles.keys()].sort()) {
+      const { cachePath } = cacheFiles.get(filename)!;
+      const { size } = await xfs.statPromise(cachePath);
+      writeNarStrings(
+        hasher,
+        "entry",
+        "(",
+        "name",
+        filename,
+        "node",
+        "(",
+        "type",
+        "regular",
+        "contents",
+      );
+      await writeNarStream(hasher, size, xfs.createReadStream(cachePath));
+      writeNarStrings(hasher, ")", ")");
+    }
+    writeNarStrings(hasher, ")");
+    hasher.end();
+    // Bit hacky, but hashers always produce a single read.
+    for await (const sha512 of hasher) {
+      combinedHash = hexToSri(sha512);
+    }
   }
-  cacheEntriesCode += `};`;
 
   // Generate Nix code for isolated builds.
-  const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
   let isolatedPackages = new Set<Package>();
   let isolatedIntegration = [];
   let isolatedCode = [];
@@ -320,6 +375,9 @@ export default async (
       PROJECT_NAME: json(projectName),
       YARN_BIN: yarnBinExpr,
       LOCKFILE: lockfileExpr,
+      INDIVIDUAL_DRVS: individualDrvs,
+      COMBINED_DRV: !individualDrvs,
+      COMBINED_HASH: combinedHash,
       CACHE_FOLDER: cacheFolderExpr,
       CACHE_ENTRIES: cacheEntriesCode,
       ISOLATED: isolatedCode.join("\n"),
@@ -350,30 +408,49 @@ export default async (
     xfs.existsSync(npath.toPortablePath(`/nix/store`))
   ) {
     await xfs.mktempPromise(async (tempDir) => {
+      const args = ["--add-fixed", "sha512"];
       const toPreload: PortablePath[] = [];
-      for (const [
-        locator,
-        { originalFilename, sha512 },
-      ] of cacheEntries.entries()) {
-        const name = sanitizeDerivationName(locator);
+      if (individualDrvs) {
+        for (const [locator, { cachePath, hash }] of cacheEntries.entries()) {
+          const name = sanitizeDerivationName(locator);
+          // Check to see if the Nix store entry already exists.
+          const storePath = computeFixedOutputStorePath(name, hash);
+          if (!xfs.existsSync(storePath)) {
+            // The nix-store command requires a correct filename on disk, so we
+            // prepare a temporary directory containing all the files to preload.
+            //
+            // Because some names may conflict (e.g. 'typescript-npm-xyz' and
+            // 'typescript-patch-xyz' both have the same derivation name), we
+            // create subdirectories based on hash.
+            const subdir = ppath.join(
+              tempDir,
+              sriToHex(hash).slice(0, 7) as Filename,
+            );
+            await xfs.mkdirPromise(subdir);
+
+            const dst = ppath.join(subdir, name as Filename);
+            await xfs.copyFilePromise(cachePath, dst);
+
+            toPreload.push(dst);
+          }
+        }
+      } else {
+        args.unshift("--recursive");
         // Check to see if the Nix store entry already exists.
-        const hash = Buffer.from(sha512, "hex");
-        const storePath = computeFixedOutputStorePath(name, `sha512`, hash);
+        const storePath = computeFixedOutputStorePath(
+          "yarn-cache",
+          combinedHash,
+          { recursive: true },
+        );
         if (!xfs.existsSync(storePath)) {
-          // The nix-store command requires a correct filename on disk, so we
-          // prepare a temporary directory containing all the files to preload.
-          //
-          // Because some names may conflict (e.g. 'typescript-npm-xyz' and
-          // 'typescript-patch-xyz' both have the same derivation name), we
-          // create subdirectories based on hash.
-          const subdir = ppath.join(tempDir, sha512.slice(0, 7) as Filename);
+          // Same as above, nix-store requires a correct filename.
+          const subdir = ppath.join(tempDir, "yarn-cache");
           await xfs.mkdirPromise(subdir);
-
-          const src = ppath.join(cache.cwd, originalFilename);
-          const dst = ppath.join(subdir, name as Filename);
-          await xfs.copyFilePromise(src, dst);
-
-          toPreload.push(dst);
+          for (const [filename, { cachePath }] of cacheFiles.entries()) {
+            const dst = ppath.join(subdir, filename);
+            await xfs.copyFilePromise(cachePath, dst);
+          }
+          toPreload.push(subdir);
         }
       }
 
@@ -382,19 +459,17 @@ export default async (
         const numToPreload = toPreload.length;
         while (toPreload.length !== 0) {
           const batch = toPreload.splice(0, 100);
-          await execUtils.execvp(
-            "nix-store",
-            ["--add-fixed", "sha512", ...batch],
-            {
-              cwd: project.cwd,
-              strict: true,
-            },
-          );
+          await execUtils.execvp("nix-store", [...args, ...batch], {
+            cwd: project.cwd,
+            strict: true,
+          });
         }
         if (numToPreload !== 0) {
           report.reportInfo(
             0,
-            `Preloaded ${numToPreload} packages into the Nix store`,
+            individualDrvs
+              ? `Preloaded ${numToPreload} packages into the Nix store`
+              : `Preloaded cache into the Nix store`,
           );
         }
       } catch (err: any) {

src/index.ts

@@ -1,7 +1,7 @@
 import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
 import { PortablePath } from "@yarnpkg/fslib";
 
-import FetchOneCommand from "./FetchOneCommand";
+import FetchCommand from "./FetchCommand";
 import InjectBuildCommand from "./InjectBuildCommand";
 import InstallBinCommand from "./InstallBinCommand";
 import generate from "./generate";
@@ -12,13 +12,14 @@ declare module "@yarnpkg/core" {
     nixExprPath: PortablePath;
     generateDefaultNix: boolean;
     enableNixPreload: boolean;
+    individualNixPackaging: boolean;
     isolatedNixBuilds: string[];
     installNixBinariesForDependencies: boolean;
   }
 }
 
 const plugin: Plugin<Hooks> = {
-  commands: [FetchOneCommand, InjectBuildCommand, InstallBinCommand],
+  commands: [FetchCommand, InjectBuildCommand, InstallBinCommand],
   hooks: {
     afterAllInstalled: async (project, opts) => {
       if (
@@ -50,6 +51,11 @@ const plugin: Plugin<Hooks> = {
       type: SettingsType.BOOLEAN,
       default: true,
     },
+    individualNixPackaging: {
+      description: `If true, generate one Nix derivation per package. If false, use a single derivation for the entire cache folder.`,
+      type: SettingsType.BOOLEAN,
+      default: false,
+    },
     isolatedNixBuilds: {
       description: `Dependencies with a build step that can be built in an isolated derivation`,
       type: SettingsType.STRING,

src/narUtils.ts

@@ -0,0 +1,51 @@
+import type { Readable, Writable } from "stream";
+
+const MAX_UINT32 = 2 ** 32 - 1;
+
+/** Write one or more NAR strings to the output. */
+export const writeNarStrings = (out: Writable, ...input: string[]) => {
+  let size = 0;
+  const bufs = input.map((str) => {
+    const buf = Buffer.from(str);
+    // TODO: Support up to MAX_SAFE_INTEGER
+    if (buf.byteLength > MAX_UINT32) {
+      throw Error(`NAR string too long: ${buf.byteLength}`);
+    }
+    size += 8 + Math.ceil(buf.byteLength / 8) * 8;
+    return buf;
+  });
+
+  const res = Buffer.alloc(size);
+  let pos = 0;
+  for (const buf of bufs) {
+    res.writeUInt32LE(buf.byteLength, pos);
+    buf.copy(res, pos + 8);
+    pos += 8 + Math.ceil(buf.byteLength / 8) * 8;
+  }
+
+  out.write(res);
+};
+
+/** Write the contents of a stream as a NAR string to the output. */
+export const writeNarStream = async (
+  out: Writable,
+  size: number,
+  input: Readable,
+) => {
+  // TODO: Support up to MAX_SAFE_INTEGER
+  if (size > MAX_UINT32) {
+    throw Error(`NAR string too long: ${size}`);
+  }
+  const sizeBuf = Buffer.alloc(8);
+  sizeBuf.writeUInt32LE(size);
+  out.write(sizeBuf);
+
+  for await (const chunk of input) {
+    out.write(chunk);
+  }
+
+  const padding = 8 - (size % 8);
+  if (padding !== 8) {
+    out.write(Buffer.alloc(padding));
+  }
+};

src/nixUtils.ts

@@ -44,13 +44,17 @@ export const encodeBase32 = (buf: Buffer) => {
  */
 export const computeFixedOutputStorePath = (
   name: string,
-  hashAlgorithm: string,
-  hash: Buffer,
-  storePath = `/nix/store` as PortablePath,
+  hash: string,
+  {
+    storePath = `/nix/store` as PortablePath,
+    recursive = false,
+  }: { storePath?: PortablePath; recursive?: boolean } = {},
 ) => {
-  const hashHex = hash.toString("hex");
+  const [hashAlgorithm, hash64] = hash.split("-");
+  const hashHex = Buffer.from(hash64, "base64").toString("hex");
 
-  const innerStr = `fixed:out:${hashAlgorithm}:${hashHex}:`;
+  const rec = recursive ? "r:" : "";
+  const innerStr = `fixed:out:${rec}${hashAlgorithm}:${hashHex}:`;
   const innerHash = computeHash(`sha256`, innerStr);
   const innerHashHex = innerHash.toString("hex");
 
@@ -71,3 +75,11 @@ export const sanitizeDerivationName = (name: string) =>
     .replace(/^\.+/, "")
     .replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
     .slice(0, 207) || "unknown";
+
+/** Convert a hexadecimal hash to an SRI hash. */
+export const hexToSri = (hash: string, algorithm = "sha512") =>
+  algorithm + "-" + Buffer.from(hash, "hex").toString("base64");
+
+/** Convert an SRI hash to a hexadecimal hash. */
+export const sriToHex = (hash: string) =>
+  Buffer.from(hash.split("-")[1], "base64").toString("hex");

src/tmpl/yarn-project.nix.in

@@ -38,25 +38,38 @@ let
     export yarn_enable_nixify=false
   '';
 
+#@@ IF COMBINED_DRV
+  cacheDrv = stdenv.mkDerivation {
+    name = "yarn-cache";
+    buildInputs = [ yarn git cacert ];
+    buildCommand = ''
+      cp --reflink=auto --recursive '${src}' ./src
+      cd ./src/
+      ${buildVars}
+      HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$out" \
+        yarn nixify fetch
+      rm $out/.gitignore
+    '';
+    outputHashMode = "recursive";
+    outputHash = "@@COMBINED_HASH@@";
+  };
+#@@ ENDIF COMBINED_DRV
+#@@ IF INDIVIDUAL_DRVS
   # Create derivations for fetching dependencies.
   cacheDrvs = let
-    builder = writeShellScript "yarn-cache-builder" ''
-      source $stdenv/setup
-      cd "$src"
+  in lib.mapAttrs (locator: { filename, hash }: stdenv.mkDerivation {
+    name = lib.strings.sanitizeDerivationName locator;
+    buildInputs = [ yarn git cacert ];
+    buildCommand = ''
+      cd '${src}'
       ${buildVars}
       HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$TMP" \
-        yarn nixify fetch-one $locator
+        yarn nixify fetch ${locator}
       # Because we change the cache dir, Yarn may generate a different name.
       mv "$TMP/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out
     '';
-  in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation {
-    inherit src builder locator;
-    name = lib.strings.sanitizeDerivationName locator;
-    buildInputs = [ yarn git cacert ];
     outputFilename = filename;
-    outputHashMode = "flat";
-    outputHashAlgo = "sha512";
-    outputHash = sha512;
+    outputHash = hash;
   }) cacheEntries;
 
   # Create a shell snippet to copy dependencies from a list of derivations.
@@ -64,6 +77,7 @@ let
     writeShellScript "collect-yarn-cache" (lib.concatMapStrings (drv: ''
       cp --reflink=auto ${drv} '${drv.outputFilename}'
     '') drvs);
+#@@ ENDIF INDIVIDUAL_DRVS
 
 #@@ IF NEED_ISOLATED_BUILD_SUPPRORT
   # Create a shell snippet to copy dependencies from a list of locators.
@@ -119,9 +133,14 @@ let
       # Copy over the Yarn cache.
       rm -fr '${cacheFolder}'
       mkdir -p '${cacheFolder}'
+#@@ IF COMBINED_DRV
+      cp --reflink=auto --recursive ${cacheDrv}/* '${cacheFolder}/'
+#@@ ENDIF COMBINED_DRV
+#@@ IF INDIVIDUAL_DRVS
       pushd '${cacheFolder}' > /dev/null
       source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
       popd > /dev/null
+#@@ ENDIF INDIVIDUAL_DRVS
 
       # Yarn may need a writable home directory.
       export yarn_global_folder="$TMP"