Merge pull request #83 from stephank/feat/isolated

Support isolated builds with a combined cache

.github/workflows/build.yml

@@ -91,7 +91,6 @@ jobs:
       - name: Test isolated builds
         run: |
           # Matches example in  ISOLATED_BUILDS.md
-          echo 'individualNixPackaging: true' >> .yarnrc.yml
           echo 'isolatedNixBuilds: ["sqlite3"]' >> .yarnrc.yml
           cat > default.nix << EOF
           { pkgs ? import <nixpkgs> { } }:
@@ -107,6 +106,13 @@ jobs:
 
           yarn add sqlite3
 
+          nix-build
+
+      - name: Test individual packaging
+        run: |
+          echo 'individualNixPackaging: true' >> .yarnrc.yml
+          yarn
+
           # This delete tests refetching a package with Nix.
           nix-store --delete /nix/store/*--yarnpkg-core-npm-*
 

ISOLATED_BUILDS.md

@@ -9,12 +9,9 @@ As an example, to create an isolated build of sqlite3, add the following to
 your `.yarnrc.yml`:
 
 ```yml
-individualNixPackaging: true
 isolatedNixBuilds: ["sqlite3"]
 ```
 
-(`individualNixPackaging` is required to use `isolatedNixBuilds`.)
-
 In your Nix expression, separate options can be set to override attributes of
 these derivations, which is often necessary to provide build inputs. For
 sqlite3, you'd do the following in your `default.nix`:

UPGRADING.md

@@ -15,6 +15,12 @@ yarn
 nix-build
 ```
 
+## Since b7200c4 (merged 2024-02-11)
+
+- `isolatedBuilds` now also works without `individualNixPackaging`.
+
+- **BREAKING**: During builds, `yarn_enable_network=0` is now set.
+
 ## Since df1c72a (merged 2024-02-06)
 
 - **BREAKING**: The plugin now generates a single derivation for the entire

src/generate.ts

@@ -52,13 +52,6 @@ export default async (
     return;
   }
 
-  // Config validation.
-  const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
-  const individualDrvs = configuration.get(`individualNixPackaging`);
-  if (isolatedBuilds.length > 0 && !individualDrvs) {
-    throw Error(`isolatedNixBuilds requires individualNixPackaging to be set`);
-  }
-
   // Determine relative paths for Nix path literals.
   const nixExprPath = configuration.get(`nixExprPath`);
 
@@ -157,6 +150,7 @@ export default async (
     hash: string;
   }
   const cacheEntries = new Map<string, CacheEntry>();
+  const individualDrvs = configuration.get(`individualNixPackaging`);
   let cacheEntriesCode = "";
   let combinedHash = "";
   if (individualDrvs) {
@@ -217,6 +211,7 @@ export default async (
   }
 
   // Generate Nix code for isolated builds.
+  const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
   let isolatedPackages = new Set<Package>();
   let isolatedIntegration = [];
   let isolatedCode = [];
@@ -332,19 +327,29 @@ export default async (
     if (!isolatedPackages.has(devirtPkg)) {
       isolatedPackages.add(devirtPkg);
 
+      const args = [
+        `pname = ${json(pkg.name)};`,
+        `version = ${json(pkg.version)};`,
+        `reference = ${json(devirtPkg.reference)};`,
+      ];
+
+      // If packaging deps individually, depend on just
+      // the deps used during this isolated build.
+      if (individualDrvs) {
         const locators = [...collectTree(pkg)]
           .sort()
           .map((v) => `${json(v)}\n`)
           .join(``);
+        if (locators) {
+          args.push(`locators = [\n${locators}];`);
+        }
+      }
 
       const overrideArg = `override${upperCamelize(pkg.name)}Attrs`;
       isolatedCode.push(
-        `${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${[
+        `${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${args.join(
-          `pname = ${json(pkg.name)};`,
+          ` `,
-          `version = ${json(pkg.version)};`,
+        )} });`,
-          `reference = ${json(devirtPkg.reference)};`,
-          `locators = [\n${locators}];`,
-        ].join(` `)} });`,
       );
     }
 
@@ -385,7 +390,7 @@ export default async (
       NEED_ISOLATED_BUILD_SUPPRORT: isolatedIntegration.length > 0,
       USES_PNP_LINKER: configuration.get("nodeLinker") === "pnp",
       USES_NM_LINKER: configuration.get("nodeLinker") === "node-modules",
-    });
+    }).replace(/\n\n\n+/g, "\n\n");
     await xfs.writeFilePromise(nixExprPath, projectExpr);
 
     // Create a wrapper if it does not exist yet.

src/tmpl/yarn-project.nix.in

@@ -24,6 +24,8 @@ let
   drvCommon = {
     # Make sure the build uses the right Node.js version everywhere.
     buildInputs = [ nodejs yarn ];
+    # All dependencies should already be cached.
+    yarn_enable_network = "0";
     # Tell node-gyp to use the provided Node.js headers for native code builds.
     npm_config_nodedir = nodejs;
   };
@@ -80,14 +82,16 @@ let
 #@@ ENDIF INDIVIDUAL_DRVS
 
 #@@ IF NEED_ISOLATED_BUILD_SUPPRORT
+#@@ IF INDIVIDUAL_DRVS
   # Create a shell snippet to copy dependencies from a list of locators.
   mkCacheBuilderForLocators = let
     pickCacheDrvs = map (locator: cacheDrvs.${locator});
   in locators:
     mkCacheBuilderForDrvs (pickCacheDrvs locators);
+#@@ ENDIF INDIVIDUAL_DRVS
 
-  # Create a derivation that builds a node-pre-gyp module in isolation.
+  # Create a derivation that builds a module in isolation.
-  mkIsolatedBuild = { pname, version, reference, locators }: stdenv.mkDerivation (drvCommon // {
+  mkIsolatedBuild = { pname, version, reference, locators ? [] }: stdenv.mkDerivation (drvCommon // {
     inherit pname version;
     dontUnpack = true;
 
@@ -98,16 +102,21 @@ let
 
     buildPhase = ''
       mkdir -p .yarn/cache
+#@@ IF COMBINED_DRV
+      cp --reflink=auto --recursive ${cacheDrv}/* .yarn/cache/
+#@@ ENDIF COMBINED_DRV
+#@@ IF INDIVIDUAL_DRVS
       pushd .yarn/cache > /dev/null
       source ${mkCacheBuilderForLocators locators}
       popd > /dev/null
+#@@ ENDIF INDIVIDUAL_DRVS
 
       echo '{ "dependencies": { "${pname}": "${reference}" } }' > package.json
       install -m 0600 ${lockfile} ./yarn.lock
       export yarn_global_folder="$TMP"
       export yarn_enable_global_cache=false
       export yarn_enable_immutable_installs=false
-      yarn --immutable-cache
+      yarn
     '';
 
     installPhase = ''