added support for native modules and patching tarballs, added node-sass example

Author: VanCoding

buildCache.nix

@@ -1,34 +1,79 @@
-{lib,runCommand,coreutils,fetchurl}:
-{packageLockPath}:
+{lib,runCommand,coreutils,fetchurl,unixtools,writeText}:
+{packageLockPath,patches}:
 let
 	json = builtins.readFile packageLockPath;
-	dependencies = (builtins.fromJSON json).dependencies;
+	packageLock = builtins.fromJSON json;
 	base64toHex = import ./base64toHex.nix;
-	getTarball = (spec:
+
+	# take a tarball and create a patched tarball in the same format
+	patchTarball = {tarball,patch}:
+		let
+			derivation = runCommand "patched-package" {buildInputs=[coreutils unixtools.xxd];} ''
+				tar -xzf ${tarball.tarball} package
+				${patch}
+				mkdir $out
+				tar -cf $out/tarball.tgz --xform 's:^\./::' package
+				sha512sum $out/tarball.tgz | cut -d " " -f 1 > $out/sha512.hex
+				xxd -r -p < $out/sha512.hex | base64  -w 0  > $out/sha512.base64
+			'';
+		in {
+			algorithm = builtins.trace "${derivation}" "sha512";
+			base64hash = builtins.readFile "${derivation}/sha512.base64";
+			hexHash = builtins.readFile "${derivation}/sha512.hex";
+			tarball = "${derivation}/tarball.tgz";
+		}
+	;
+	
+	# get a flat list of all depencency specs
+	getFlatSpecs = dependencies:
+		builtins.concatLists (lib.attrsets.mapAttrsToList (name: spec: [spec]++(if spec ? dependencies then getFlatSpecs spec.dependencies else [])) dependencies)
+	;
+
+	# convert a dependency spec into a tarball with additional information
+	getTarballFromSpec = spec:
 		let
 			parts = lib.strings.splitString "-" spec.integrity;
+		in rec {
 			algorithm = builtins.elemAt parts 0;
 			base64hash = builtins.elemAt parts 1;
 			hexHash = base64toHex base64hash;
 			tarball = fetchurl {
 				url = spec.resolved;
 				"${algorithm}" = hexHash;
 			};
-			dependencies = if spec ? dependencies then getTarballs spec.dependencies else [];
+		}
+	;
+
+	# get the command to create the cache entry for this tarball
+	getCommandFromTarball = {algorithm,hexHash,tarball,...}:
+		let
 			folder1 = builtins.substring 0 2 hexHash;
 			folder2 = builtins.substring 2 2 hexHash;
 			fileName = builtins.substring 4 ((builtins.stringLength hexHash)-4) hexHash;
 			folderPath = "$out/_cacache/content-v2/${algorithm}/${folder1}/${folder2}";
-			command = "mkdir -p ${folderPath} && cd ${folderPath} && ln -sf ${tarball} ${fileName}";
 		in
-			[command]++dependencies
-	);
-	getTarballs = (dependencies:
-		builtins.concatLists (lib.attrsets.mapAttrsToList (name: spec: getTarball spec) dependencies)
-	);	
-	tarballs = getTarballs dependencies;
-	command = builtins.concatStringsSep "\n" tarballs;
-in
-	runCommand "cache" {buildInputs = [coreutils];} ''
-		${command}
-	''
+			"mkdir -p ${folderPath} && cd ${folderPath} && ln -sf ${tarball} ${fileName}"
+	;
+
+	# flat list of all dependency specs
+	dependencies = getFlatSpecs packageLock.dependencies;
+
+	# flat list of all tarballs
+	tarballs = builtins.map getTarballFromSpec dependencies;
+
+	# an index of all tarballs by their base64 hash
+	tarballsByBase64Hash = builtins.listToAttrs (builtins.map (tarball: {name = tarball.base64hash; value = tarball;}) tarballs);
+
+	# an index of all patched tarballs by their base64 hash
+	patchedTarballs = lib.attrsets.mapAttrs (hash: patch: patchTarball {tarball = tarballsByBase64Hash."${hash}"; inherit patch;}) patches;
+
+	# all commands to create the cache
+	commands = builtins.concatStringsSep "\n" (builtins.map getCommandFromTarball (builtins.map (tarball: if builtins.hasAttr tarball.base64hash patchedTarballs then patchedTarballs."${tarball.base64hash}" else tarball) tarballs));
+in {
+	cache = runCommand "cache" {buildInputs = [coreutils];} ''
+		${commands}
+	'';
+
+	# when we patch packages, we also need to return a new package-lock.json because the hashes changed
+	packageLock = writeText "package-lock.json" ( builtins.foldl' (json: hash: builtins.replaceStrings [hash] [patchedTarballs."${hash}".base64hash] json) json (builtins.attrNames patches));
+}

buildNodeModules.nix

@@ -1,17 +1,30 @@
-{lib,stdenv,runCommand,coreutils,fetchurl,nodejs}:
-{directory,buildInputs?[]}:
+{lib,stdenv,runCommand,coreutils,fetchurl,nodejs,writeText,unixtools}:
+{directory,buildInputs?[],preInstallScript?"",postInstallScript?"",patches?{}}:
 let
-    buildCache = import ./buildCache.nix {inherit lib runCommand coreutils fetchurl;};
-    cache = buildCache {packageLockPath = "${directory}/package-lock.json";};
-    derivation = stdenv.mkDerivation {
+    # import the function to build the cache
+    buildCache = import ./buildCache.nix {inherit lib runCommand coreutils fetchurl writeText unixtools;};
+
+    # build the cache
+    cache = buildCache {packageLockPath = "${directory}/package-lock.json"; inherit patches;};
+
+    # get the Node.js sources needed by native modules
+    nodeSources = runCommand "node-sources" {} ''
+        tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
+        mv node-* $out
+    '';
+
+    # build the node_modules directory
+    derivation = builtins.trace "${cache.packageLock}" stdenv.mkDerivation {
         pname = "node_modules";
         version = "";
         buildInputs=[nodejs coreutils]++buildInputs;
         phases = ["buildPhase" "patchPhase" "fixupPhase" ];
-        buildPhase = ''
-            cp ${directory}/package-lock.json ./package-lock.json
+        buildPhase =  ''
+            cp ${cache.packageLock} package-lock.json
             cp ${directory}/package.json ./package.json
-            npm ci --cache ${cache}
+            ${preInstallScript}
+            npm ci --cache ${cache.cache} --nodedir=${nodeSources}
+            ${postInstallScript}
             mkdir $out
             mv node_modules $out/node_modules/
         '';

examples/express-app/default.nix

@@ -1,10 +1,20 @@
-{nodejs,node_modules,runCommand}:
-runCommand "express-app" {buildInputs=[nodejs];} ''
-    mkdir $out
-    cp ${./index.js} $out/index.js
-    ln -s "${node_modules}" $out/node_modules
-    mkdir $out/bin
-    echo '#!node
-    require("../index.js")' > $out/bin/express-app
-    chmod +x $out/bin/express-app
-''
+{pkgs?import<nixpkgs> {}}: with pkgs;
+let
+    buildNodeModules = callPackage ../../buildNodeModules.nix {};
+in rec {
+    node_modules = buildNodeModules {directory = ./.;};
+    app = runCommand "express-app" {buildInputs=[nodejs];} ''
+        mkdir $out
+        cp ${./index.js} $out/index.js
+        ln -s "${node_modules}" $out/node_modules
+        mkdir $out/bin
+        echo '#!node
+        require("../index.js")' > $out/bin/express-app
+        chmod +x $out/bin/express-app
+    '';
+    shell = mkShell {
+        shellHook = ''
+            ln -sf ${node_modules} node_modules
+        '';
+    };
+}

examples/express-app/node_modules.nix

@@ -1,15 +1 @@
-with import <nixpkgs> {};
-
-let
-    buildNodeModules = callPackage ../../buildNodeModules.nix {};
-    node_modules = import ./node_modules.nix {inherit buildNodeModules;};
-    express-app = callPackage ./default.nix {inherit nodejs node_modules;};
-in
-    mkShell {
-        buildInputs = [
-            express-app
-        ];
-        shellHook = ''
-            ln -s ${node_modules} node_modules
-        '';
-    }
+(import ./default.nix {}).shell

examples/express-app/shell.nix

@@ -0,0 +1,15 @@
+{pkgs?import<nixpkgs> {}}:with pkgs;
+let
+    buildNodeModules = callPackage ../../buildNodeModules.nix {};
+in rec {
+    node_modules = buildNodeModules {directory = ./.;buildInputs=[python];};
+    lib = runCommand "sass-lib" {} ''
+        export PATH=$PATH:${node_modules}/.bin
+        node-sass ${./src} -o $out
+    '';
+    shell = mkShell {
+        shellHook = ''
+            ln -sf ${node_modules} node_modules
+        '';
+    };
+}