Isolated builds of dependencies

README.md

@@ -82,15 +82,17 @@ Some examples of what's possible:
 
 let
 
-  # Example of providing a different source tree.
-  src = pkgs.lib.cleanSource ./.;
-
   project = pkgs.callPackage ./yarn-project.nix {
 
     # Example of selecting a specific version of Node.js.
     nodejs = pkgs.nodejs-14_x;
 
-  } src;
+  } {
+
+    # Example of providing a different source tree.
+    src = pkgs.lib.cleanSource ./.;
+
+  }
 
 in project.overrideAttrs (oldAttrs: {
 

build.js

@@ -10,6 +10,8 @@ const IS_PROD = process.argv[2] === `-p`;
 const EXTERNALS = [
   `@yarnpkg/core`,
   `@yarnpkg/fslib`,
+  `@yarnpkg/parsers`,
+  `@yarnpkg/plugin-patch`,
   `@yarnpkg/plugin-pnp`,
   `clipanion`,
   `crypto`,

package.json

@@ -17,6 +17,8 @@
     "@yarnpkg/cli": "^2.2.0",
     "@yarnpkg/core": "^2.2.0",
     "@yarnpkg/fslib": "^2.2.0",
+    "@yarnpkg/parsers": "^2.2.0",
+    "@yarnpkg/plugin-patch": "^2.1.1",
     "@yarnpkg/plugin-pnp": "^2.2.0",
     "babel-loader": "^8.1.0",
     "clipanion": "^2.4.4",

src/FetchOneCommand.ts

@@ -4,16 +4,16 @@ import {
   Cache,
   CommandContext,
   Configuration,
-  LocatorHash,
   Project,
   StreamReport,
+  structUtils,
 } from "@yarnpkg/core";
 
 // Internal command that fetches a single locator.
 // Used from within Nix to build the cache for the project.
 export default class FetchOneCommand extends Command<CommandContext> {
   @Command.String()
-  locatorHash: string = ``;
+  locator: string = ``;
 
   @Command.Path(`nixify`, `fetch-one`)
   async execute() {
@@ -29,11 +29,10 @@ export default class FetchOneCommand extends Command<CommandContext> {
     const report = await StreamReport.start(
       { configuration, stdout: this.context.stdout },
       async (report) => {
-        const pkg = project.originalPackages.get(
+        const { locatorHash } = structUtils.parseLocator(this.locator, true);
-          this.locatorHash as LocatorHash
+        const pkg = project.originalPackages.get(locatorHash);
-        );
         if (!pkg) {
-          report.reportError(0, `Invalid locator hash`);
+          report.reportError(0, `Invalid locator`);
           return;
         }
 

src/InjectBuildCommand.ts

@@ -0,0 +1,146 @@
+import { Command } from "clipanion";
+import { PortablePath, ppath, xfs } from "@yarnpkg/fslib";
+import { createHash } from "crypto";
+import { parseSyml, stringifySyml } from "@yarnpkg/parsers";
+
+import {
+  CommandContext,
+  Configuration,
+  execUtils,
+  Locator,
+  LocatorHash,
+  Project,
+  StreamReport,
+  structUtils,
+} from "@yarnpkg/core";
+
+// Internal command that injects an isolated build inside a Nix build.
+export default class InjectBuildCommand extends Command<CommandContext> {
+  @Command.String()
+  locator: string = ``;
+  @Command.String()
+  source: string = ``;
+  @Command.String()
+  installLocation: string = ``;
+
+  @Command.Path(`nixify`, `inject-build`)
+  async execute() {
+    const configuration = await Configuration.find(
+      this.context.cwd,
+      this.context.plugins
+    );
+    const { project } = await Project.find(configuration, this.context.cwd);
+
+    const report = await StreamReport.start(
+      { configuration, stdout: this.context.stdout },
+      async (report) => {
+        // To find virtualized packages, we need parse the lockfile.
+        await project.resolveEverything({ report, lockfileOnly: true });
+
+        const locator = structUtils.parseLocator(this.locator, true);
+        const pkg = project.storedPackages.get(locator.locatorHash);
+        if (!pkg) {
+          report.reportError(0, `Invalid locator`);
+          return;
+        }
+
+        // Copy over the build directory.
+        // Can't use xfs.copyPromise, because it doesn't know how to deal with
+        // the read-only permissions of our source path in the Nix store.
+        const installLocation = ppath.join(
+          project.cwd,
+          this.installLocation as PortablePath
+        );
+        await xfs.mkdirpPromise(ppath.dirname(installLocation));
+        await execUtils.execvp("cp", ["-R", this.source, installLocation], {
+          cwd: project.cwd,
+          strict: true,
+        });
+        await execUtils.execvp("chmod", ["-R", "u+w", installLocation], {
+          cwd: project.cwd,
+          strict: true,
+        });
+
+        // Imitate Project: generate the global hash
+        const globalHashGenerator = createHash(`sha512`);
+        globalHashGenerator.update(process.versions.node);
+
+        configuration.triggerHook(
+          (hooks) => {
+            return hooks.globalHashGeneration;
+          },
+          project,
+          (data: Buffer | string) => {
+            globalHashGenerator.update(`\0`);
+            globalHashGenerator.update(data);
+          }
+        );
+
+        const globalHash = globalHashGenerator.digest(`hex`);
+
+        // Imitate Project: generate a package hash
+        const packageHashMap = new Map<LocatorHash, string>();
+        const getBaseHash = (locator: Locator) => {
+          let hash = packageHashMap.get(locator.locatorHash);
+          if (typeof hash !== `undefined`) return hash;
+
+          const pkg = project.storedPackages.get(locator.locatorHash);
+          if (typeof pkg === `undefined`)
+            throw new Error(
+              `Assertion failed: The package should have been registered`
+            );
+
+          const builder = createHash(`sha512`);
+          builder.update(locator.locatorHash);
+
+          // To avoid the case where one dependency depends on itself somehow
+          packageHashMap.set(locator.locatorHash, `<recursive>`);
+
+          for (const descriptor of pkg.dependencies.values()) {
+            const resolution = project.storedResolutions.get(
+              descriptor.descriptorHash
+            );
+            if (typeof resolution === `undefined`)
+              throw new Error(
+                `Assertion failed: The resolution (${structUtils.prettyDescriptor(
+                  project.configuration,
+                  descriptor
+                )}) should have been registered`
+              );
+
+            const dependency = project.storedPackages.get(resolution);
+            if (typeof dependency === `undefined`)
+              throw new Error(
+                `Assertion failed: The package should have been registered`
+              );
+
+            builder.update(getBaseHash(dependency));
+          }
+
+          hash = builder.digest(`hex`);
+          packageHashMap.set(locator.locatorHash, hash);
+
+          return hash;
+        };
+
+        // Imitate Project: create a build hash that Yarn accepts
+        const buildHash = createHash(`sha512`)
+          .update(globalHash)
+          .update(getBaseHash(pkg))
+          .update(installLocation)
+          .digest(`hex`);
+
+        // Update build state. The way we do this is crude, but we run
+        // `yarn install` later, which should clean it up again.
+        const bstatePath: PortablePath = configuration.get(`bstatePath`);
+        const bstate: { [key: string]: string } = xfs.existsSync(bstatePath)
+          ? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`))
+          : {};
+        bstate[pkg.locatorHash] = buildHash;
+        await xfs.writeFilePromise(bstatePath, stringifySyml(bstate));
+      }
+    );
+
+    return report.exitCode();
+  }
+}

src/InstallBinCommand.ts

@@ -11,6 +11,7 @@ import {
 
 import binWrapperNodeModulesTmpl from "./tmpl/bin-wrapper-node-modules.sh.in";
 import binWrapperPnpTmpl from "./tmpl/bin-wrapper-pnp.sh.in";
+import { renderTmpl } from "./textUtils";
 
 const supportedLinkers = [`pnp`, `node-modules`];
 
@@ -59,22 +60,24 @@ export default class InstallBinCommand extends Command<CommandContext> {
           let script;
           switch (nodeLinker) {
             case `pnp`:
-              script = binWrapperPnpTmpl
+              script = renderTmpl(binWrapperPnpTmpl, {
-                .replace(`@@NODE_PATH@@`, process.execPath)
+                NODE_PATH: process.execPath,
-                .replace(`@@PNP_PATH@@`, pnpPath)
+                PNP_PATH: pnpPath,
-                .replace(`@@SCRIPT_PATH@@`, scriptPath);
+                SCRIPT_PATH: scriptPath,
+              });
               break;
             case `node-modules`:
-              script = binWrapperNodeModulesTmpl
+              script = renderTmpl(binWrapperNodeModulesTmpl, {
-                .replace(`@@NODE_PATH@@`, process.execPath)
+                NODE_PATH: process.execPath,
-                .replace(`@@SCRIPT_PATH@@`, scriptPath);
+                SCRIPT_PATH: scriptPath,
+              });
               break;
             default:
-              throw Error(`Invalid nodeLinker ${nodeLinker}`);
+              throw Error(`Assertion failed: Invalid nodeLinker ${nodeLinker}`);
           }
 
-          xfs.writeFileSync(binPath, script);
+          await xfs.writeFilePromise(binPath, script);
-          xfs.chmodSync(binPath, 0o755);
+          await xfs.chmodPromise(binPath, 0o755);
         }
       }
     );

src/generate.ts

@@ -1,10 +1,17 @@
 import { Filename, npath, PortablePath, ppath, xfs } from "@yarnpkg/fslib";
-import { computeFixedOutputStorePath } from "./nixUtils";
+import { parseSyml } from "@yarnpkg/parsers";
+import { patchUtils } from "@yarnpkg/plugin-patch";
+import {
+  computeFixedOutputStorePath,
+  sanitizeDerivationName,
+} from "./nixUtils";
+import { json, indent, renderTmpl, upperCamelize } from "./textUtils";
 
 import {
   Cache,
   execUtils,
   LocatorHash,
+  Package,
   Project,
   Report,
   structUtils,
@@ -14,21 +21,6 @@ import defaultExprTmpl from "./tmpl/default.nix.in";
 import projectExprTmpl from "./tmpl/yarn-project.nix.in";
 import { tmpdir } from "os";
 
-interface CacheEntry {
-  name: string;
-  filename: Filename;
-  sha512: string;
-  locatorHash: LocatorHash;
-}
-
-const cacheEntryToNix = (entry: CacheEntry) =>
-  [
-    `name = ${JSON.stringify(entry.name)};`,
-    `filename = ${JSON.stringify(entry.filename)};`,
-    `sha512 = ${JSON.stringify(entry.sha512)};`,
-    `locatorHash = ${JSON.stringify(entry.locatorHash)};`,
-  ].join(` `);
-
 // Generator function that runs after `yarn install`.
 export default async (project: Project, cache: Cache, report: Report) => {
   const { configuration, cwd } = project;
@@ -39,7 +31,7 @@ export default async (project: Project, cache: Cache, report: Report) => {
   //
   // On macOS at least, we also need to get the real path of the OS temp dir,
   // because it goes through a symlink.
-  const tempDir = xfs.realpathSync(npath.toPortablePath(tmpdir()));
+  const tempDir = await xfs.realpathPromise(npath.toPortablePath(tmpdir()));
   if (project.cwd.startsWith(tempDir)) {
     report.reportInfo(
       0,
@@ -81,11 +73,26 @@ export default async (project: Project, cache: Cache, report: Report) => {
     }
   }
 
+  // Determine relative paths for Nix path literals.
+  const nixExprPath = configuration.get(`nixExprPath`);
+  const lockfileFilename = configuration.get(`lockfileFilename`);
+  const lockfileRel = ppath.relative(
+    ppath.dirname(nixExprPath),
+    lockfileFilename
+  );
+  const yarnPathRel = ppath.relative(ppath.dirname(nixExprPath), yarnPath);
+
   // Build a list of cache entries so Nix can fetch them.
-  let cacheEntries: CacheEntry[] = [];
+  // TODO: See if we can use Nix fetchurl for npm: dependencies.
-  const cacheFiles = new Set(xfs.readdirSync(cache.cwd));
+  interface CacheEntry {
+    filename: Filename;
+    sha512: string;
+  }
+  const cacheEntries: Map<string, CacheEntry> = new Map();
+
+  const cacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
   for (const pkg of project.storedPackages.values()) {
-    const { version, locatorHash } = pkg;
+    const { locatorHash } = pkg;
     const checksum = project.storedChecksums.get(locatorHash);
     if (!checksum) continue;
 
@@ -95,39 +102,192 @@ export default async (project: Project, cache: Cache, report: Report) => {
     const filename = ppath.basename(cachePath);
     if (!cacheFiles.has(filename)) continue;
 
-    let name = structUtils.slugifyIdent(pkg).replace(/^@/, "_at_");
+    const locatorStr = structUtils.stringifyLocator(pkg);
-    if (version) {
+    const sha512 = checksum.split(`/`).pop()!;
-      name += `-${version}`;
+    cacheEntries.set(locatorStr, { filename, sha512 });
   }
 
-    const sha512 = checksum.split(`/`).pop()!;
+  let cacheEntriesCode = `cacheEntries = {\n`;
-    cacheEntries.push({ name, filename, sha512, locatorHash });
+  for (const [locatorStr, entry] of cacheEntries) {
+    cacheEntriesCode += `${json(locatorStr)} = { ${[
+      `filename = ${json(entry.filename)};`,
+      `sha512 = ${json(entry.sha512)};`,
+    ].join(` `)} };\n`;
+  }
+  cacheEntriesCode += `};`;
+
+  // Generate Nix code for isolated builds.
+  const isolatedBuilds: string[] = configuration.get(`isolatedNixBuilds`);
+  let isolatedPackages = new Set<Package>();
+  let isolatedIntegration = [];
+  let isolatedCode = [];
+
+  const nodeLinker = configuration.get(`nodeLinker`);
+  const pnpUnpluggedFolder = configuration.get(`pnpUnpluggedFolder`);
+  const bstatePath: PortablePath = configuration.get(`bstatePath`);
+  const bstate: { [key: string]: string } = xfs.existsSync(bstatePath)
+    ? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`))
+    : {};
+
+  const collectTree = (pkg: Package, out: Set<string> = new Set()) => {
+    const locatorStr = structUtils.stringifyLocator(pkg);
+    if (cacheEntries.has(locatorStr)) {
+      out.add(locatorStr);
+    }
+
+    if (structUtils.isVirtualLocator(pkg)) {
+      const devirtPkg = project.storedPackages.get(
+        structUtils.devirtualizeLocator(pkg).locatorHash
+      );
+      if (!devirtPkg) {
+        throw Error(
+          `Assertion failed: The locator should have been registered`
+        );
+      }
+
+      collectTree(devirtPkg, out);
+    }
+
+    if (pkg.reference.startsWith("patch:")) {
+      const depatchPkg = project.storedPackages.get(
+        patchUtils.parseLocator(pkg).sourceLocator.locatorHash
+      );
+      if (!depatchPkg) {
+        throw Error(
+          `Assertion failed: The locator should have been registered`
+        );
+      }
+
+      collectTree(depatchPkg, out);
+    }
+
+    for (const dependency of pkg.dependencies.values()) {
+      const resolution = project.storedResolutions.get(
+        dependency.descriptorHash
+      );
+      if (!resolution) {
+        throw Error(
+          "Assertion failed: The descriptor should have been registered"
+        );
+      }
+
+      const depPkg = project.storedPackages.get(resolution);
+      if (!depPkg) {
+        throw Error(
+          `Assertion failed: The locator should have been registered`
+        );
+      }
+
+      collectTree(depPkg, out);
+    }
+
+    return out;
+  };
+
+  for (const locatorHash of Object.keys(bstate)) {
+    const pkg = project.storedPackages.get(locatorHash as LocatorHash);
+    if (!pkg) {
+      throw Error(`Assertion failed: The locator should have been registered`);
+    }
+
+    // TODO: Better options for matching.
+    if (!isolatedBuilds.includes(pkg.name)) {
+      continue;
+    }
+
+    // TODO: We can't currently support the node-modules linker, because it
+    // always clears build state.
+    let installLocation: PortablePath;
+    switch (nodeLinker) {
+      case `pnp`:
+        installLocation = ppath.relative(
+          project.cwd,
+          ppath.join(
+            pnpUnpluggedFolder,
+            structUtils.slugifyLocator(pkg),
+            structUtils.getIdentVendorPath(pkg)
+          )
+        );
+        break;
+      default:
+        throw Error(
+          `The nodeLinker ${nodeLinker} is not supported for isolated Nix builds`
+        );
+    }
+
+    // Virtualization typically happens when the package has peer dependencies,
+    // and thus it depends on context how the package is built. But we
+    // eliminate that context, so devirtualize.
+    let devirtPkg = pkg;
+    if (structUtils.isVirtualLocator(devirtPkg)) {
+      const { locatorHash } = structUtils.devirtualizeLocator(devirtPkg);
+      const pkg = project.storedPackages.get(locatorHash);
+      if (!pkg) {
+        throw Error(
+          `Assertion failed: The locator should have been registered`
+        );
+      }
+      devirtPkg = pkg;
+    }
+
+    const buildLocatorStr = structUtils.stringifyLocator(devirtPkg);
+    const injectLocatorStr = structUtils.stringifyLocator(pkg);
+    const isolatedProp = `isolated.${json(buildLocatorStr)}`;
+
+    if (!isolatedPackages.has(devirtPkg)) {
+      isolatedPackages.add(devirtPkg);
+
+      const locators = [...collectTree(pkg)]
+        .sort()
+        .map((v) => `${json(v)}\n`)
+        .join(``);
+
+      const overrideArg = `override${upperCamelize(pkg.name)}Attrs`;
+      isolatedCode.push(
+        `${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${[
+          `pname = ${json(pkg.name)};`,
+          `version = ${json(pkg.version)};`,
+          `locators = [\n${locators}];`,
+        ].join(` `)} });`
+      );
+    }
+
+    if (isolatedIntegration.length === 0) {
+      isolatedIntegration.push("# Copy in isolated builds.");
+    }
+    isolatedIntegration.push(
+      `echo 'injecting build for ${pkg.name}'`,
+      `yarn nixify inject-build \\`,
+      `  ${json(injectLocatorStr)} \\`,
+      `  $\{${isolatedProp}} \\`,
+      `  ${json(installLocation)}`
+    );
+  }
+  if (isolatedIntegration.length > 0) {
+    isolatedIntegration.push(`echo 'running yarn install'`);
   }
 
   // Render the Nix expression.
   const ident = project.topLevelWorkspace.manifest.name;
   const projectName = ident ? structUtils.stringifyIdent(ident) : `workspace`;
-  const projectExpr = projectExprTmpl
+  const projectExpr = renderTmpl(projectExprTmpl, {
-    .replace(`@@PROJECT_NAME@@`, JSON.stringify(projectName))
+    PROJECT_NAME: json(projectName),
-    .replace(`@@YARN_PATH@@`, JSON.stringify(yarnPath))
+    YARN_PATH: yarnPathRel,
-    .replace(`@@CACHE_FOLDER@@`, JSON.stringify(cacheFolder))
+    LOCKFILE: lockfileRel,
-    .replace(
+    CACHE_FOLDER: json(cacheFolder),
-      `@@CACHE_ENTRIES@@`,
+    CACHE_ENTRIES: cacheEntriesCode,
-      `[\n` +
+    ISOLATED: isolatedCode.join("\n"),
-        cacheEntries
+    ISOLATED_INTEGRATION: indent("      ", isolatedIntegration.join("\n")),
-          .map((entry) => `    { ${cacheEntryToNix(entry)} }\n`)
+    NEED_ISOLATED_BUILD_SUPPRORT: isolatedIntegration.length > 0,
-          .sort()
+  });
-          .join(``) +
+  await xfs.writeFilePromise(configuration.get(`nixExprPath`), projectExpr);
-        `  ]`
-    );
-  xfs.writeFileSync(configuration.get(`nixExprPath`), projectExpr);
 
   // Create a wrapper if it does not exist yet.
   if (configuration.get(`generateDefaultNix`)) {
     const defaultExprPath = ppath.join(cwd, `default.nix` as Filename);
     const flakeExprPath = ppath.join(cwd, `flake.nix` as Filename);
     if (!xfs.existsSync(defaultExprPath) && !xfs.existsSync(flakeExprPath)) {
-      xfs.writeFileSync(defaultExprPath, defaultExprTmpl);
+      await xfs.writeFilePromise(defaultExprPath, defaultExprTmpl);
       report.reportInfo(
         0,
         `A minimal default.nix was created. You may want to customize it.`
@@ -142,7 +302,8 @@ export default async (project: Project, cache: Cache, report: Report) => {
   ) {
     await xfs.mktempPromise(async (tempDir) => {
       const toPreload: PortablePath[] = [];
-      for (const { name, filename, sha512 } of cacheEntries) {
+      for (const [locator, { filename, sha512 }] of cacheEntries.entries()) {
+        const name = sanitizeDerivationName(locator);
         // Check to see if the Nix store entry already exists.
         const hash = Buffer.from(sha512, "hex");
         const storePath = computeFixedOutputStorePath(name, `sha512`, hash);

src/index.ts

@@ -1,10 +1,12 @@
-import FetchOneCommand from "./FetchOneCommand";
-import InstallBinCommand from "./InstallBinCommand";
-import generate from "./generate";
 import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
 
+import FetchOneCommand from "./FetchOneCommand";
+import InjectBuildCommand from "./InjectBuildCommand";
+import InstallBinCommand from "./InstallBinCommand";
+import generate from "./generate";
+
 const plugin: Plugin<Hooks> = {
-  commands: [FetchOneCommand, InstallBinCommand],
+  commands: [FetchOneCommand, InjectBuildCommand, InstallBinCommand],
   hooks: {
     afterAllInstalled: async (project, opts) => {
       if (
@@ -36,6 +38,12 @@ const plugin: Plugin<Hooks> = {
       type: SettingsType.BOOLEAN,
       default: true,
     },
+    isolatedNixBuilds: {
+      description: `Dependencies with a build step that can be built in an isolated derivation`,
+      type: SettingsType.STRING,
+      default: [],
+      isArray: true,
+    },
   },
 };
 

src/nixUtils.ts

@@ -60,3 +60,14 @@ export const computeFixedOutputStorePath = (
 
   return ppath.join(storePath, `${outerHash32}-${name}` as Filename);
 };
+
+/**
+ * Creates a valid derivation name from a potentially invalid one.
+ *
+ * Matches lib.strings.sanitizeDerivationName in Nixpkgs.
+ */
+export const sanitizeDerivationName = (name: string) =>
+  name
+    .replace(/^\.+/, "")
+    .replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
+    .slice(0, 207) || "unknown";

src/textUtils.ts

@@ -0,0 +1,67 @@
+export const json = JSON.stringify;
+
+/**
+ * Capitalize the first character.
+ */
+export const ucfirst = (value: string) =>
+  value.slice(0, 1).toUpperCase() + value.slice(1);
+
+/**
+ * Convert to a camel-case starting with uppercase.
+ *
+ * Example: `one two three` => `OneTwoThree`
+ */
+export const upperCamelize = (name: string) =>
+  name
+    .split(/[^a-zA-Z0-9]+/g)
+    .filter((x) => x)
+    .map((v) => ucfirst(v))
+    .join("");
+
+/**
+ * Add a prefix to every line in some text.
+ */
+export const indent = (
+  prefix: string,
+  text: string,
+  includeEmptyLines = false
+): string =>
+  text
+    .split("\n")
+    .map((line: string) => (line || includeEmptyLines ? prefix + line : line))
+    .join("\n");
+
+/**
+ * Basic templating rendering.
+ *
+ * String values in `vars` will be used for simple substitution
+ * of `@@KEY@@`, while boolean values will be used for
+ * conditional sections of code between `#@@ IF KEY` and `#@@
+ * ENDIF KEY`.
+ */
+export const renderTmpl = (
+  tmpl: string,
+  vars: { [name: string]: string | boolean }
+): string => {
+  let result = tmpl;
+  for (const [name, value] of Object.entries(vars)) {
+    if (typeof value === "string") {
+      result = result.replace(`@@${name}@@`, value);
+    }
+    if (typeof value === "boolean") {
+      const lines = result.split("\n");
+      const startIdx = lines.indexOf(`#@@ IF ${name}`);
+      const endIdx = lines.indexOf(`#@@ ENDIF ${name}`);
+      if (startIdx !== -1 && endIdx > startIdx) {
+        if (value) {
+          lines.splice(endIdx, 1);
+          lines.splice(startIdx, 1);
+        } else {
+          lines.splice(startIdx, endIdx - startIdx + 1);
+        }
+        result = lines.join("\n");
+      }
+    }
+  }
+  return result;
+};

src/tmpl/default.nix.in

@@ -3,4 +3,4 @@
 
 { pkgs ? import <nixpkgs> { } }:
 
-pkgs.callPackage ./yarn-project.nix { } ./.
+pkgs.callPackage ./yarn-project.nix { } { src = ./.; }

src/tmpl/yarn-project.nix.in

@@ -1,59 +1,101 @@
 # This file is generated by running "yarn install" inside your project.
 # Manual changes might be lost - proceed with caution!
 
-{ lib, coreutils, nodejs, stdenv, writeText }: src:
+{ lib, nodejs, stdenv, fetchurl, writeText }:
+{ src, overrideAttrs ? null, ... } @ args:
 
 let
 
-  yarnPath = @@YARN_PATH@@;
+  yarnPath = ./@@YARN_PATH@@;
+  lockfile = ./@@LOCKFILE@@;
   cacheFolder = @@CACHE_FOLDER@@;
-  cacheEntries = @@CACHE_ENTRIES@@;
 
-  # Fetch a single dependency.
+  # Call overrideAttrs on a derivation if a function is provided.
-  fetchOne = let
+  optionalOverride = fn: drv:
+    if fn == null then drv else drv.overrideAttrs fn;
+
+  # Common attributes between Yarn derivations.
+  drvCommon = {
+    # Make sure the build uses the right Node.js version everywhere.
+    buildInputs = [ nodejs ];
+    # Tell node-gyp to use the provided Node.js headers for native code builds.
+    npm_config_nodedir = nodejs;
+    # Tell node-pre-gyp to never fetch binaries / always build from source.
+    npm_config_build_from_source = "true";
+    # Defines the shell alias to run Yarn.
+    postHook = ''
+      yarn() {
+        CI=1 node "${yarnPath}" "$@"
+      }
+    '';
+  };
+
+  # Create derivations for fetching dependencies.
+  cacheDrvs = let
     builder = builtins.toFile "builder.sh" ''
       source $stdenv/setup
       cd "$src"
       HOME="$TMP" yarn_cache_folder="$TMP" CI=1 \
-        node '${yarnPath}' nixify fetch-one $locatorHash
+        node '${yarnPath}' nixify fetch-one $locator
       mv "$TMP/$outputFilename" $out
     '';
-  in { name, filename, sha512, locatorHash }: stdenv.mkDerivation {
+  in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation {
-    inherit name src builder locatorHash;
+    inherit src builder locator;
+    name = lib.strings.sanitizeDerivationName locator;
     buildInputs = [ nodejs ];
     outputFilename = filename;
     outputHashMode = "flat";
     outputHashAlgo = "sha512";
     outputHash = sha512;
-  };
+  }) cacheEntries;
 
-  # Shell snippet to collect all project dependencies.
+  # Create a shell snippet to copy dependencies from a list of derivations.
-  collectCacheScript = writeText "collect-cache.sh" (
+  mkCacheBuilderForDrvs = drvs:
-    lib.concatMapStrings (args: ''
+    writeText "collect-cache.sh" (lib.concatMapStrings (drv: ''
-      cp ${fetchOne args} '${args.filename}'
+      cp ${drv} '${drv.outputFilename}'
-    '') cacheEntries
+    '') drvs);
-  );
 
-in stdenv.mkDerivation {
+#@@ IF NEED_ISOLATED_BUILD_SUPPRORT
-  name = @@PROJECT_NAME@@;
+  # Create a shell snippet to copy dependencies from a list of locators.
+  mkCacheBuilderForLocators = let
+    pickCacheDrvs = map (locator: cacheDrvs.${locator});
+  in locators:
+    mkCacheBuilderForDrvs (pickCacheDrvs locators);
+
+  # Create a derivation that builds a node-pre-gyp module in isolation.
+  mkIsolatedBuild = { pname, version, locators }: stdenv.mkDerivation (drvCommon // {
+    inherit pname version;
+    phases = [ "buildPhase" "installPhase" ];
+
+    buildPhase = ''
+      mkdir -p .yarn/cache
+      pushd .yarn/cache > /dev/null
+      source ${mkCacheBuilderForLocators locators}
+      popd > /dev/null
+
+      echo '{ "dependencies": { "${pname}": "${version}" } }' > package.json
+      install -m 0600 ${lockfile} ./yarn.lock
+      yarn --immutable-cache
+    '';
+
+    installPhase = ''
+      unplugged=( .yarn/unplugged/${pname}-*/node_modules/* )
+      if [[ ! -e "''${unplugged[@]}" ]]; then
+        echo >&2 "Could not find the unplugged path for ${pname}"
+        exit 1
+      fi
+
+      mv "$unplugged" $out
+    '';
+  });
+#@@ ENDIF NEED_ISOLATED_BUILD_SUPPRORT
+
+  # Main project derivation.
+  project = stdenv.mkDerivation (drvCommon // {
     inherit src;
-
+    name = @@PROJECT_NAME@@;
     # Disable Nixify plugin to save on some unnecessary processing.
     yarn_enable_nixify = "false";
-  # Tell node-gyp to use the provided Node.js headers for native code builds.
-  npm_config_nodedir = nodejs;
-  # Tell node-pre-gyp to never fetch binaries / always build from source.
-  npm_config_build_from_source = "true";
-
-  # Make sure the build uses the right Node.js version everywhere.
-  buildInputs = [ nodejs ];
-
-  # Defines the shell alias to run Yarn.
-  postHook = ''
-    yarn() {
-      CI=1 node "$NIX_YARN_PATH" "$@"
-    }
-  '';
 
     configurePhase = ''
       runHook preConfigure
@@ -62,11 +104,10 @@ in stdenv.mkDerivation {
       rm -fr '${cacheFolder}'
       mkdir -p '${cacheFolder}'
       pushd '${cacheFolder}' > /dev/null
-    source ${collectCacheScript}
+      source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
       popd > /dev/null
 
-    # Store the absolute path to Yarn for the 'yarn' alias.
+@@ISOLATED_INTEGRATION@@
-    export NIX_YARN_PATH="$(readlink -f '${yarnPath}')"
 
       # Run normal Yarn install to complete dependency installation.
       yarn install --immutable --immutable-cache
@@ -88,9 +129,6 @@ in stdenv.mkDerivation {
       mv $PWD "$out/libexec/$sourceRoot"
       cd "$out/libexec/$sourceRoot"
 
-    # Update the path to Yarn.
-    export NIX_YARN_PATH="$(readlink -f '${yarnPath}')"
-
       # Invoke a plugin internal command to setup binaries.
       yarn nixify install-bin $out/bin
 
@@ -100,4 +138,8 @@ in stdenv.mkDerivation {
     passthru = {
       inherit nodejs;
     };
-}
+  });
+
+@@CACHE_ENTRIES@@
+@@ISOLATED@@
+in optionalOverride overrideAttrs project

yarn.lock

@@ -3105,6 +3105,8 @@ typescript@4.2.4:
     "@yarnpkg/cli": ^2.2.0
     "@yarnpkg/core": ^2.2.0
     "@yarnpkg/fslib": ^2.2.0
+    "@yarnpkg/parsers": ^2.2.0
+    "@yarnpkg/plugin-patch": ^2.1.1
     "@yarnpkg/plugin-pnp": ^2.2.0
     babel-loader: ^8.1.0
     clipanion: ^2.4.4