diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b71811..1917362 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -91,7 +91,6 @@ jobs: - name: Test isolated builds run: | # Matches example in ISOLATED_BUILDS.md - echo 'individualNixPackaging: true' >> .yarnrc.yml echo 'isolatedNixBuilds: ["sqlite3"]' >> .yarnrc.yml cat > default.nix << EOF { pkgs ? import { } }: @@ -107,6 +106,13 @@ jobs: yarn add sqlite3 + nix-build + + - name: Test individual packaging + run: | + echo 'individualNixPackaging: true' >> .yarnrc.yml + yarn + # This delete tests refetching a package with Nix. nix-store --delete /nix/store/*--yarnpkg-core-npm-* diff --git a/ISOLATED_BUILDS.md b/ISOLATED_BUILDS.md index b72e821..47f2dc9 100644 --- a/ISOLATED_BUILDS.md +++ b/ISOLATED_BUILDS.md @@ -9,12 +9,9 @@ As an example, to create an isolated build of sqlite3, add the following to your `.yarnrc.yml`: ```yml -individualNixPackaging: true isolatedNixBuilds: ["sqlite3"] ``` -(`individualNixPackaging` is required to use `isolatedNixBuilds`.) - In your Nix expression, separate options can be set to override attributes of these derivations, which is often necessary to provide build inputs. For sqlite3, you'd do the following in your `default.nix`: diff --git a/src/generate.ts b/src/generate.ts index 9bc79fe..08511ca 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -52,13 +52,6 @@ export default async ( return; } - // Config validation. - const isolatedBuilds = configuration.get(`isolatedNixBuilds`); - const individualDrvs = configuration.get(`individualNixPackaging`); - if (isolatedBuilds.length > 0 && !individualDrvs) { - throw Error(`isolatedNixBuilds requires individualNixPackaging to be set`); - } - // Determine relative paths for Nix path literals. const nixExprPath = configuration.get(`nixExprPath`); @@ -157,6 +150,7 @@ export default async ( hash: string; } const cacheEntries = new Map(); + const individualDrvs = configuration.get(`individualNixPackaging`); let cacheEntriesCode = ""; let combinedHash = ""; if (individualDrvs) { @@ -217,6 +211,7 @@ export default async ( } // Generate Nix code for isolated builds. + const isolatedBuilds = configuration.get(`isolatedNixBuilds`); let isolatedPackages = new Set(); let isolatedIntegration = []; let isolatedCode = []; @@ -332,19 +327,29 @@ export default async ( if (!isolatedPackages.has(devirtPkg)) { isolatedPackages.add(devirtPkg); - const locators = [...collectTree(pkg)] - .sort() - .map((v) => `${json(v)}\n`) - .join(``); + const args = [ + `pname = ${json(pkg.name)};`, + `version = ${json(pkg.version)};`, + `reference = ${json(devirtPkg.reference)};`, + ]; + + // If packaging deps individually, depend on just + // the deps used during this isolated build. + if (individualDrvs) { + const locators = [...collectTree(pkg)] + .sort() + .map((v) => `${json(v)}\n`) + .join(``); + if (locators) { + args.push(`locators = [\n${locators}];`); + } + } const overrideArg = `override${upperCamelize(pkg.name)}Attrs`; isolatedCode.push( - `${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${[ - `pname = ${json(pkg.name)};`, - `version = ${json(pkg.version)};`, - `reference = ${json(devirtPkg.reference)};`, - `locators = [\n${locators}];`, - ].join(` `)} });`, + `${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${args.join( + ` `, + )} });`, ); } diff --git a/src/tmpl/yarn-project.nix.in b/src/tmpl/yarn-project.nix.in index 5f929ea..e98fc1c 100644 --- a/src/tmpl/yarn-project.nix.in +++ b/src/tmpl/yarn-project.nix.in @@ -24,6 +24,8 @@ let drvCommon = { # Make sure the build uses the right Node.js version everywhere. buildInputs = [ nodejs yarn ]; + # All dependencies should already be cached. + yarn_enable_network = "0"; # Tell node-gyp to use the provided Node.js headers for native code builds. npm_config_nodedir = nodejs; }; @@ -80,14 +82,16 @@ let #@@ ENDIF INDIVIDUAL_DRVS #@@ IF NEED_ISOLATED_BUILD_SUPPRORT +#@@ IF INDIVIDUAL_DRVS # Create a shell snippet to copy dependencies from a list of locators. mkCacheBuilderForLocators = let pickCacheDrvs = map (locator: cacheDrvs.${locator}); in locators: mkCacheBuilderForDrvs (pickCacheDrvs locators); +#@@ ENDIF INDIVIDUAL_DRVS - # Create a derivation that builds a node-pre-gyp module in isolation. - mkIsolatedBuild = { pname, version, reference, locators }: stdenv.mkDerivation (drvCommon // { + # Create a derivation that builds a module in isolation. + mkIsolatedBuild = { pname, version, reference, locators ? [] }: stdenv.mkDerivation (drvCommon // { inherit pname version; dontUnpack = true; @@ -98,16 +102,21 @@ let buildPhase = '' mkdir -p .yarn/cache +#@@ IF COMBINED_DRV + cp --reflink=auto --recursive ${cacheDrv}/* .yarn/cache/ +#@@ ENDIF COMBINED_DRV +#@@ IF INDIVIDUAL_DRVS pushd .yarn/cache > /dev/null source ${mkCacheBuilderForLocators locators} popd > /dev/null +#@@ ENDIF INDIVIDUAL_DRVS echo '{ "dependencies": { "${pname}": "${reference}" } }' > package.json install -m 0600 ${lockfile} ./yarn.lock export yarn_global_folder="$TMP" export yarn_enable_global_cache=false export yarn_enable_immutable_installs=false - yarn --immutable-cache + yarn ''; installPhase = ''