Granular fetching

src/index.ts

@@ -7,11 +7,11 @@ import {
   CommandContext,
   Configuration,
   Hooks,
+  LocatorHash,
   Plugin,
   Project,
   Report,
   StreamReport,
-  execUtils,
   structUtils,
 } from "@yarnpkg/core";
 
@@ -20,7 +20,7 @@ import defaultExprTmpl from "./default.nix.in";
 import projectExprTmpl from "./yarn-project.nix.in";
 
 // Generator function that runs after `yarn install`.
-const generate = async (project: Project, report: Report) => {
+const generate = async (project: Project, cache: Cache, report: Report) => {
   const { configuration, cwd } = project;
   const yarnPathAbs = configuration.get(`yarnPath`);
   const cacheFolderAbs = configuration.get(`cacheFolder`);
@@ -99,25 +99,25 @@ const generate = async (project: Project, report: Report) => {
     }
   }
 
-  // Build the Nix output-hash by hashing the Yarn cache folder. The
+  // Build a list of cache entries so Nix can fetch them.
-  // derivation should build the exact same.
+  let cacheEntries = [];
-  let cacheHash = ``;
+  const cacheFiles = new Set(xfs.readdirSync(cache.cwd));
-  try {
+  for (const pkg of project.storedPackages.values()) {
-    const hasherResult = await execUtils.execvp(
+    const checksum = project.storedChecksums.get(pkg.locatorHash);
-      `nix-hash`,
+    if (!checksum) continue;
-      [`--type`, `sha256`, `--base32`, cacheFolder],
+
-      { cwd, encoding: `utf8`, strict: true }
+    const cachePath = cache.getLocatorPath(pkg, checksum);
-    );
+    if (!cachePath) continue;
-    cacheHash = hasherResult.stdout.trim();
+
-  } catch (err) {
+    const filename = ppath.basename(cachePath);
-    if (err.code === `ENOENT`) {
+    if (!cacheFiles.has(filename)) continue;
-      report.reportWarning(
+
-        0,
+    const sha512 = checksum.split(`/`).pop();
-        `No Nix installation found - yarn-project.nix will not be updated`
+    cacheEntries.push([
-      );
+      `filename = ${JSON.stringify(filename)};`,
-    } else {
+      `sha512 = ${JSON.stringify(sha512)};`,
-      throw err;
+      `locator-hash = ${JSON.stringify(pkg.locatorHash)};`,
-    }
+    ]);
   }
 
   // Render the Nix expression.
@@ -126,14 +126,21 @@ const generate = async (project: Project, report: Report) => {
   const projectExpr = projectExprTmpl
     .replace(`@@PROJECT_NAME@@`, JSON.stringify(projectName))
     .replace(`@@CACHE_FOLDER@@`, JSON.stringify(cacheFolder))
-    .replace(`@@CACHE_HASH@@`, JSON.stringify(cacheHash))
+    .replace(
+      `@@CACHE_ENTRIES@@`,
+      `[\n` +
+        [...cacheEntries]
+          .map((entry) => `    { ${entry.join(` `)} }\n`)
+          .join(``) +
+        `  ]`
+    )
     .replace(`@@YARN_PATH@@`, JSON.stringify(yarnPath))
     .replace(
       `@@YARN_CLOSURE_ENTRIES@@`,
-      `[ ` +
+      `[\n` +
         [...yarnClosureEntries]
-          .map((entry) => JSON.stringify(entry))
+          .map((entry) => `    ${JSON.stringify(entry)}\n`)
-          .join(` `) +
+          .join(``) +
         `  ]`
     );
   const projectExprPath = ppath.join(cwd, `yarn-project.nix` as Filename);
@@ -150,40 +157,41 @@ const generate = async (project: Project, report: Report) => {
   }
 };
 
-// Internal command that does just the fetch part of `yarn install`.
+// Internal command that fetches a single locator.
-// Used inside the Nix offline-cache derivation to build the cache.
+// Used from within Nix to build the cache for the project.
-class BuildCacheCommand extends Command<CommandContext> {
+class FetchOneCommand extends Command<CommandContext> {
   @Command.String()
-  out: string = ``;
+  locatorHash: string = ``;
 
-  @Command.Path(`nixify`, `build-cache`)
+  @Command.Path(`nixify`, `fetch-one`)
   async execute() {
     const configuration = await Configuration.find(
       this.context.cwd,
       this.context.plugins
     );
-
-    // Overwrite the cache directory to our output directory.
-    configuration.use(
-      `<nixify>`,
-      { cacheFolder: this.out },
-      configuration.projectCwd!,
-      { overwrite: true }
-    );
-
     const { project } = await Project.find(configuration, this.context.cwd);
     const cache = await Cache.find(configuration);
 
-    // Run resolution and fetch steps.
+    const fetcher = configuration.makeFetcher();
+
     const report = await StreamReport.start(
       { configuration, stdout: this.context.stdout },
       async (report) => {
-        await report.startTimerPromise(`Resolution step`, () =>
+        const pkg = project.originalPackages.get(
-          project.resolveEverything({ report, lockfileOnly: true })
+          this.locatorHash as LocatorHash
-        );
-        await report.startTimerPromise(`Fetch step`, () =>
-          project.fetchEverything({ cache, report })
         );
+        if (!pkg) {
+          report.reportError(0, `Invalid locator hash`);
+          return;
+        }
+
+        await fetcher.fetch(pkg, {
+          checksums: project.storedChecksums,
+          project,
+          cache,
+          fetcher,
+          report,
+        });
       }
     );
 
@@ -228,11 +236,11 @@ class InstallBinCommand extends Command<CommandContext> {
 }
 
 const plugin: Plugin<Hooks> = {
-  commands: [BuildCacheCommand, FetchLocatorCommand, InstallBinCommand],
+  commands: [FetchOneCommand, InstallBinCommand],
   hooks: {
     afterAllInstalled: async (project, opts) => {
       if (opts.persistProject !== false) {
-        await generate(project, opts.report);
+        await generate(project, opts.cache, opts.report);
       }
     },
   },

src/yarn-project.nix.in

@@ -10,7 +10,7 @@ let
   # Variables provided by the generator.
   project-name = @@PROJECT_NAME@@;
   cache-folder = @@CACHE_FOLDER@@;
-  cache-hash = @@CACHE_HASH@@;
+  cache-entries = @@CACHE_ENTRIES@@;
   yarn-path = @@YARN_PATH@@;
   yarn-closure-entries = @@YARN_CLOSURE_ENTRIES@@;
 
@@ -31,30 +31,44 @@ let
       elem "${type}:${srcRel path}" yarn-closure-entries;
   };
 
-  # Build just the cache for the project.
+  # Fetch a single dependency.
-  offline-cache = stdenv.mkDerivation {
+  fetch = { filename, sha512, locator-hash }: stdenv.mkDerivation {
-    name = "${project-name}-offline-cache";
+    name = replaceStrings [ "@" ] [ "-" ] filename;
     buildInputs = [ nodejs ];
-    builder = writeText "builder.sh" ''
+    builder = builtins.toFile "builder.sh" ''
       source $stdenv/setup
-      cd '${yarn-closure}'
 
       # Yarn may need a writable home directory for the global cache mirror.
       # TODO: Can't disable the mirror, because it changes cache filenames.
-      export HOME="$TEMP"
+      export HOME="$PWD"
+
+      # Fetch into the build directory.
+      export yarn_cache_folder="$PWD"
 
       # Setup to environment so we can run Yarn.
+      pushd '${yarn-closure}' > /dev/null
+
       export NIX_YARN_PATH="$(readlink -f '${yarn-path}')"
       ${yarn-alias}
 
       # Invoke a plugin internal command to build the cache.
-      yarn nixify build-cache $out
+      yarn nixify fetch-one '${locator-hash}'
+
+      popd > /dev/null
+
+      # The cache file that was just fetched is our output.
+      mv '${filename}' $out
     '';
-    outputHashMode = "recursive";
+    outputHashMode = "flat";
-    outputHashAlgo = "sha256";
+    outputHashAlgo = "sha512";
-    outputHash = cache-hash;
+    outputHash = sha512;
   };
 
+  # Shell snippet to collect all project dependencies.
+  collect-cache = concatMapStrings (args: ''
+    cp ${fetch args} './${args.filename}'
+  '') cache-entries;
+
 in stdenv.mkDerivation {
   name = project-name;
   src = ./.;
@@ -74,11 +88,12 @@ in stdenv.mkDerivation {
     runHook preConfigure
 
     # Copy over the Yarn cache.
-    # TODO: Can we do without the copy somehow?
+    # TODO: Can we do without the copy somehow? Links don't work.
     rm -fr '${cache-folder}'
-    mkdir -p "$(dirname '${cache-folder}')"
+    mkdir -p '${cache-folder}'
-    cp -r '${offline-cache}' '${cache-folder}'
+    pushd '${cache-folder}' > /dev/null
-    chmod -R u+w '${cache-folder}'
+    ${collect-cache}
+    popd > /dev/null
 
     # Store the absolute path to Yarn for the 'yarn' alias.
     export NIX_YARN_PATH="$(readlink -f '${yarn-path}')"