From cdefe26742f442351e73ce0f7caa3f559be32dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 7 May 2024 07:40:29 +0200 Subject: [PATCH] load database with checksum again but do the same in disko-install --- disko-install | 21 ++++++++++++++- disko-install.nix | 22 ++++++++++++---- docs/disko-install.md | 3 ++- install-cli.nix | 3 +++ lib/closure-info.nix | 45 --------------------------------- lib/make-disk-image.nix | 11 +++----- tests/disko-install/default.nix | 3 ++- 7 files changed, 48 insertions(+), 60 deletions(-) delete mode 100644 lib/closure-info.nix diff --git a/disko-install b/disko-install index db551964..19a13bc8 100755 --- a/disko-install +++ b/disko-install @@ -207,11 +207,13 @@ main() { --arg diskMappings "$(serialiaseArrayToNix diskMappings)" \ --argstr extraSystemConfig "$extraSystemConfig" \ -A installToplevel \ + -A closureInfo \ -A "$diskoAttr") IFS=$'\n' mapfile -t artifacts <<<"$outputs" nixos_system=${artifacts[0]} - disko_script=${artifacts[1]} + closure_info=${artifacts[1]} + disko_script=${artifacts[2]} if [[ -n ${dry_run-} ]]; then echo "Would run: $disko_script" @@ -227,6 +229,23 @@ main() { cp -ar "$source" "$mountPoint/$destination" done + # nix copy uses up a lot of memory and we work around issues with incorrect checksums in our store + # that can be caused by using closureInfo in combination with multiple builders and non-deterministic builds. + # Therefore if we have a blank store, we copy the store paths and registration from the closureInfo. + if [[ ! -d "${mountPoint}/nix/store" ]]; then + export NIX_STATE_DIR=${mountPoint}/nix/var/nix + echo "Copying store paths" >&2 + + if [ -t 1 ]; then + xargs -I% xcp --recursive % "${mountPoint}/nix/store" < "${closure_info}/store-paths" + else + xargs -I% xcp --recursive --no-progress % "${mountPoint}/nix/store" < "${closure_info}/store-paths" + fi + echo "Loading nix database" >&2 + nix-store --load-db < "${closure_info}/registration" + unset NIX_STATE_DIR + fi + nixos-install --no-root-password --system "$nixos_system" --root "$mountPoint" } diff --git a/disko-install.nix b/disko-install.nix index 48ff6aca..dbf124e2 100644 --- a/disko-install.nix +++ b/disko-install.nix @@ -1,11 +1,16 @@ -{ stdenvNoCC, makeWrapper, lib }: +{ + stdenvNoCC, + makeWrapper, + lib, + coreutils, + xcp, + nixos-install-tools, +}: stdenvNoCC.mkDerivation { name = "disko-install"; src = ./.; - nativeBuildInputs = [ - makeWrapper - ]; + nativeBuildInputs = [ makeWrapper ]; installPhase = '' mkdir -p $out/bin $out/share/disko cp -r install-cli.nix $out/share/disko @@ -14,7 +19,14 @@ stdenvNoCC.mkDerivation { -e "s|#!/usr/bin/env.*|#!/usr/bin/env bash|" \ disko-install > $out/bin/disko-install chmod 755 $out/bin/disko-install - wrapProgram $out/bin/disko-install + wrapProgram $out/bin/disko-install \ + --prefix PATH : "${ + lib.makeBinPath [ + coreutils + xcp + nixos-install-tools + ] + }" ''; meta = with lib; { description = "Disko and nixos-install in one command"; diff --git a/docs/disko-install.md b/docs/disko-install.md index bec4f983..4476bb98 100644 --- a/docs/disko-install.md +++ b/docs/disko-install.md @@ -197,9 +197,10 @@ Add this to your flake.nix output: { pkgs, self, ... }: let dependencies = [ - pkgs.stdenv.drvPath self.nixosConfigurations.your-machine.config.system.build.toplevel self.nixosConfigurations.your-machine.config.system.build.diskoScript + self.nixosConfigurations.your-machine.pkgs.stdenv.drvPath + (self.nixosConfigurations.your-machine.pkgs.closureInfo { rootPaths = [ ]; }).drvPath ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; diff --git a/install-cli.nix b/install-cli.nix index d1cd148d..c173bf81 100644 --- a/install-cli.nix +++ b/install-cli.nix @@ -62,5 +62,8 @@ let in { installToplevel = installSystem.config.system.build.toplevel; + closureInfo = installSystem.pkgs.closureInfo { + rootPaths = [ installSystem.config.system.build.toplevel ]; + }; inherit (diskoSystem.config.system.build) formatScript mountScript diskoScript; } diff --git a/lib/closure-info.nix b/lib/closure-info.nix deleted file mode 100644 index e40fe18b..00000000 --- a/lib/closure-info.nix +++ /dev/null @@ -1,45 +0,0 @@ -# This is a modified version of the closure-info derivation from nixpkgs. -# Unlike the original, it does not include hashes in the registration file, -# which might be incorrect if a build is not binary reproducible. - -# This derivation builds two files containing information about the -# closure of 'rootPaths': $out/store-paths contains the paths in the -# closure, and $out/registration contains a file suitable for use with -# "nix-store --register-validity". - -{ stdenv, coreutils, jq }: - -{ rootPaths }: - -assert builtins.langVersion >= 5; - -stdenv.mkDerivation { - name = "closure-info"; - - __structuredAttrs = true; - - exportReferencesGraph.closure = rootPaths; - - preferLocalBuild = true; - - nativeBuildInputs = [ coreutils jq ]; - - empty = rootPaths == []; - - buildCommand = - '' - out=''${outputs[out]} - - mkdir $out - - if [[ -n "$empty" ]]; then - echo 0 > $out/total-nar-size - touch $out/registration $out/store-paths - else - jq -r ".closure | map(.narSize) | add" < "$NIX_ATTRS_JSON_FILE" > $out/total-nar-size - jq -r '.closure | map([.path, "", (.references | length)] + .references) | add | map("\(.)\n") | add' < "$NIX_ATTRS_JSON_FILE" | head -n -1 > $out/registration - jq -r '.closure[].path' < "$NIX_ATTRS_JSON_FILE" > $out/store-paths - fi - - ''; -} diff --git a/lib/make-disk-image.nix b/lib/make-disk-image.nix index 87120d5b..c9b42f90 100644 --- a/lib/make-disk-image.nix +++ b/lib/make-disk-image.nix @@ -28,6 +28,7 @@ let systemdMinimal nix util-linux + xcp ] ++ nixosConfig.config.disko.extraDependencies; preVM = '' ${lib.concatMapStringsSep "\n" (disk: "truncate -s ${disk.imageSize} ${disk.name}.raw") (lib.attrValues nixosConfig.config.disko.devices.disk)} @@ -39,7 +40,7 @@ let ${extraPostVM} ''; - closureInfo = (pkgs.callPackage ./closure-info.nix { }) { + closureInfo = pkgs.closureInfo { rootPaths = [ systemToInstall.config.system.build.toplevel ]; }; partitioner = '' @@ -63,15 +64,11 @@ let installer = '' # populate nix db, so nixos-install doesn't complain export NIX_STATE_DIR=${systemToInstall.config.disko.rootMountPoint}/nix/var/nix - # We have to use fakeroot here, because nix tries to chown files in the original store. - # fakeroot will make override all chown calls to no-ops. - echo "Registering store paths..." - ${pkgs.fakeroot}/bin/fakeroot nix-store --register-validity --reregister < ${closureInfo}/registration + nix-store --load-db < "${closureInfo}/registration" # We copy files with cp because `nix copy` seems to have a large memory leak - echo "Copying store paths..." mkdir -p ${systemToInstall.config.disko.rootMountPoint}/nix/store - xargs cp -r --target-directory=${systemToInstall.config.disko.rootMountPoint}/nix/store < ${closureInfo}/store-paths + xargs -I % xcp --recursive % ${systemToInstall.config.disko.rootMountPoint}/nix/store < ${closureInfo}/store-paths ${systemToInstall.config.system.build.nixos-install}/bin/nixos-install --root ${systemToInstall.config.disko.rootMountPoint} --system ${systemToInstall.config.system.build.toplevel} --keep-going --no-channel-copy -v --no-root-password --option binary-caches "" umount -Rv ${systemToInstall.config.disko.rootMountPoint} diff --git a/tests/disko-install/default.nix b/tests/disko-install/default.nix index 6360de4e..ac85b66f 100644 --- a/tests/disko-install/default.nix +++ b/tests/disko-install/default.nix @@ -3,7 +3,8 @@ let disko-install = pkgs.callPackage ../../disko-install.nix { }; dependencies = [ - pkgs.stdenv.drvPath + self.nixosConfigurations.testmachine.pkgs.stdenv.drvPath + (self.nixosConfigurations.testmachine.pkgs.closureInfo { rootPaths = []; }).drvPath self.nixosConfigurations.testmachine.config.system.build.toplevel self.nixosConfigurations.testmachine.config.system.build.diskoScript ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);