From 9a44c6d4d020035f25baf74b1f7986b3221a274e Mon Sep 17 00:00:00 2001 From: Daniel Hill Date: Fri, 25 Nov 2022 12:47:29 +1300 Subject: [PATCH] nix: overhaul build system. Removed outdated overlay. Simply build tooling using bingenHook and propagated*Inputs Signed-off-by: Daniel Hill --- .editorconfig | 6 ++ .gitignore | 5 ++ base.nix | 65 +++++++++++++++ binary.nix | 129 +++++++++++++++++++++++++++++ default.nix | 138 +++---------------------------- flake.lock | 31 +++---- flake.nix | 120 ++++++--------------------- nix/bcachefs-kernel.nix | 34 -------- nix/bcachefs.rev.sha256 | 1 - nix/fetchnix.nix | 48 ----------- nix/nixpkgs.json | 5 -- nix/nixpkgs.nix | 9 -- nix/overlay.nix | 28 ------- nix/update-nixpkgs.sh | 32 ------- rust-src/bch_bindgen/default.nix | 83 +++++++------------ rust-src/mount/default.nix | 46 ++++------- shell.nix | 5 +- 17 files changed, 305 insertions(+), 480 deletions(-) create mode 100644 .editorconfig create mode 100644 base.nix create mode 100644 binary.nix delete mode 100644 nix/bcachefs-kernel.nix delete mode 100644 nix/bcachefs.rev.sha256 delete mode 100644 nix/fetchnix.nix delete mode 100644 nix/nixpkgs.json delete mode 100644 nix/nixpkgs.nix delete mode 100644 nix/overlay.nix delete mode 100755 nix/update-nixpkgs.sh diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..86ba213 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,6 @@ +indent_style = tab +indent_size = 8 + +[*.nix] +indent_style = space +indent_size = 2 diff --git a/.gitignore b/.gitignore index b1c03cd..7ba6add 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,23 @@ bcachefs bcachefs.5 .* *.o +*.so *.d *.a +/rust-src/mount/result +/rust-src/bch_bindgen/result tags TAGS cscope* bcachefs-tools +compile_commands.json tests/test_helper tests/__pycache__/ # dot-files that we don't want to ignore !.gitignore !.travis.yml +!.editorconfig mount/target mount.bcachefs diff --git a/base.nix b/base.nix new file mode 100644 index 0000000..29c86ac --- /dev/null +++ b/base.nix @@ -0,0 +1,65 @@ +{ lib +, doCheck ? true +, stdenvNoCC +, callPackage +, nixosTests +, autoPatchelfHook +, binary +, mount +, versionString ? "0.1" +, inShell ? false +, debugMode ? inShell +, testWithValgrind ? true +, fuseSupport ? false +, fuse3 ? null }: + +stdenvNoCC.mkDerivation { + pname = "bcachefs-tools"; + + version = "v0.1-flake-${versionString}"; + + nativeBuildInputs = [ + binary + mount + ]; + + buildInputs = mount.propagatedBuildInputs; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir $out + mkdir $out/bin + mkdir $out/lib + mkdir $out/share + mkdir $out/etc + cp -pr "${binary}/bin/"* $out/bin + cp -pr "${binary}/lib/"* $out/lib + cp -pr "${binary}/share/"* $out/share + cp -pr "${binary}/etc/"* $out/etc + cp -pr "${mount}/bin/"* $out/bin/ + chmod u+w $out/bin/* + patchelf --add-rpath $out/lib $out/bin/bcachefs-mount + ln -s "$out/bin/bcachefs-mount" "$out/bin/mount.bcachefs" + ln -s "$out/bin" "$out/sbin" + ''; + doCheck = doCheck; # needs bcachefs module loaded on builder + + passthru = { + tests = { + smoke-test = nixosTests.bcachefs; + }; + }; + + enableParallelBuilding = true; + meta = with lib; { + description = "Userspace tools for bcachefs"; + homepage = http://bcachefs.org; + license = licenses.gpl2; + platforms = platforms.linux; + maintainers = + [ "Kent Overstreet " + ]; + + }; +} diff --git a/binary.nix b/binary.nix new file mode 100644 index 0000000..e20e503 --- /dev/null +++ b/binary.nix @@ -0,0 +1,129 @@ +{ lib +, stdenv +, pkg-config +, attr +, libuuid +, libsodium +, keyutils + +, liburcu +, zlib +, libaio +, udev +, zstd +, lz4 + +, python39 +, python39Packages +, docutils +, nixosTests + +, versionString ? "0.1" +, doCheck ? true +, inShell ? false +, debugMode ? inShell + +, testWithValgrind ? true +, valgrind + +, fuseSupport ? false +, fuse3 ? null }: + +assert fuseSupport -> fuse3 != null; +assert testWithValgrind -> valgrind != null; +stdenv.mkDerivation { + pname = "bcachefs-bin"; + + version = "v0.1-flake-${versionString}"; + VERSION = "v0.1-flake-${versionString}"; + + src = (lib.cleanSource (builtins.path { name = "bcachefs-tools-src"; path = ./. ;} )); + + postPatch = "patchShebangs --build doc/macro2rst.py"; + + propagatedNativeBuildInputs = [ + # used to find dependencies + ## see ./INSTALL + pkg-config + ]; + + propagatedBuildInputs = [ + # bcachefs explicit dependencies + ## see ./INSTALL + libaio + + # libblkid + keyutils # libkeyutils + lz4 # liblz4 + + libsodium + liburcu + libuuid + zstd # libzstd + zlib # zlib1g + + # unspecified dependencies + attr + udev + ]; + + buildInputs = [ + # documentation depenedencies + docutils + python39Packages.pygments + ] ++ (lib.optional fuseSupport fuse3) + ++ (lib.optional testWithValgrind valgrind); + + makeFlags = [ + "PREFIX=${placeholder "out"}" + ] ++ lib.optional debugMode "EXTRA_CFLAGS=-ggdb"; + + installFlags = [ + "INITRAMFS_DIR=${placeholder "out"}/etc/initramfs-tools" + ]; + + doCheck = doCheck; # needs bcachefs module loaded on builder + + checkInputs = [ + python39Packages.pytest + python39Packages.pytest-xdist + ] ++ lib.optional testWithValgrind valgrind; + + checkFlags = [ + "BCACHEFS_TEST_USE_VALGRIND=${if testWithValgrind then "yes" else "no"}" + # cannot escape spaces within make flags, quotes are stripped + "PYTEST_CMD=pytest" # "PYTEST_ARGS='-n4 --version'" + ]; + + preCheck = + '' + makeFlagsArray+=(PYTEST_ARGS="--verbose -n2") + '' + + lib.optionalString fuseSupport '' + rm tests/test_fuse.py + ''; + + dontStrip = debugMode; + passthru = { + bcachefs_revision = let + file = builtins.readFile ./.bcachefs_revision; + removeLineFeeds = str: lib.lists.foldr (lib.strings.removeSuffix) str ["\r" "\n"]; + in removeLineFeeds file; + + tests = { + smoke-test = nixosTests.bcachefs; + }; + }; + + enableParallelBuilding = true; + meta = with lib; { + description = "Userspace tools for bcachefs"; + homepage = http://bcachefs.org; + license = licenses.gpl2; + platforms = platforms.linux; + maintainers = + [ "Kent Overstreet " + ]; + + }; +} diff --git a/default.nix b/default.nix index a693194..2cccff2 100644 --- a/default.nix +++ b/default.nix @@ -1,128 +1,10 @@ -{ lib -, doCheck ? true -, stdenv -, pkg-config -, attr -, libuuid -, libsodium -, keyutils - -, liburcu -, zlib -, libaio -, udev -, zstd -, lz4 - -, python39 -, python39Packages -, docutils -, nixosTests - -, versionString ? "0.1" - -, inShell ? false -, debugMode ? inShell - -, testWithValgrind ? true -, valgrind - -, fuseSupport ? false -, fuse3 ? null }: - -assert fuseSupport -> fuse3 != null; -assert testWithValgrind -> valgrind != null; -stdenv.mkDerivation { - pname = "bcachefs-tools"; - - version = "v0.1-flake-${versionString}"; - VERSION = "v0.1-flake-${versionString}"; - - src = (lib.cleanSource (builtins.path { name = "bcachefs-tools-src"; path = ./. ;} )); - - postPatch = "patchShebangs --build doc/macro2rst.py"; - - nativeBuildInputs = [ - # used to find dependencies - ## see ./INSTALL - pkg-config - ]; - buildInputs = [ - # bcachefs explicit dependencies - ## see ./INSTALL - libaio - - # libblkid - keyutils # libkeyutils - lz4 # liblz4 - - libsodium - liburcu - libuuid - zstd # libzstd - zlib # zlib1g - valgrind - - # unspecified dependencies - attr - udev - - # documentation depenedencies - docutils - python39Packages.pygments - ] ++ (lib.optional fuseSupport fuse3) - ++ (lib.optional testWithValgrind valgrind) ; - - makeFlags = [ - "PREFIX=${placeholder "out"}" - ] ++ lib.optional debugMode "EXTRA_CFLAGS=-ggdb"; - - installFlags = [ - "INITRAMFS_DIR=${placeholder "out"}/etc/initramfs-tools" - ]; - - doCheck = doCheck; # needs bcachefs module loaded on builder - - checkInputs = [ - python39Packages.pytest - python39Packages.pytest-xdist - ] ++ lib.optional testWithValgrind valgrind; - - checkFlags = [ - "BCACHEFS_TEST_USE_VALGRIND=${if testWithValgrind then "yes" else "no"}" - # cannot escape spaces within make flags, quotes are stripped - "PYTEST_CMD=pytest" # "PYTEST_ARGS='-n4 --version'" - ]; - - preCheck = - '' - makeFlagsArray+=(PYTEST_ARGS="--verbose -n2") - '' + - lib.optionalString fuseSupport '' - rm tests/test_fuse.py - ''; - - dontStrip = debugMode; - passthru = { - bcachefs_revision = let - file = builtins.readFile ./.bcachefs_revision; - removeLineFeeds = str: lib.lists.foldr (lib.strings.removeSuffix) str ["\r" "\n"]; - in removeLineFeeds file; - - tests = { - smoke-test = nixosTests.bcachefs; - }; - }; - - enableParallelBuilding = true; - meta = with lib; { - description = "Userspace tools for bcachefs"; - homepage = http://bcachefs.org; - license = licenses.gpl2; - platforms = platforms.linux; - maintainers = - [ "Kent Overstreet " - ]; - - }; -} +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).defaultNix diff --git a/flake.lock b/flake.lock index 2c9c15b..a708f2a 100644 --- a/flake.lock +++ b/flake.lock @@ -1,27 +1,28 @@ { "nodes": { - "filter": { + "flake-compat": { + "flake": false, "locked": { - "lastModified": 1620202920, - "narHash": "sha256-BOkm3eKT45Dk4NNxJT0xL9NnyYeZcF+t79zPnJkggac=", - "owner": "numtide", - "repo": "nix-filter", - "rev": "3c9e33ed627e009428197b07216613206f06ed80", + "lastModified": 1668681692, + "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "009399224d5e398d03b22badca40a37ac85412a1", "type": "github" }, "original": { - "owner": "numtide", - "repo": "nix-filter", + "owner": "edolstra", + "repo": "flake-compat", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1633351077, - "narHash": "sha256-z38JG4Bb0GtM1aF1pANVdp1dniMP23Yb3HnRoJRy2uU=", + "lastModified": 1669320964, + "narHash": "sha256-EBFw+ge12Pcr3qCk8If3/eMBAoQLR7ytndXZoRevUtM=", "owner": "nixos", "repo": "nixpkgs", - "rev": "14aef06d9b3ad1d07626bdbb16083b83f92dc6c1", + "rev": "27ccd29078f974ddbdd7edc8e38c8c8ae003c877", "type": "github" }, "original": { @@ -33,18 +34,18 @@ }, "root": { "inputs": { - "filter": "filter", + "flake-compat": "flake-compat", "nixpkgs": "nixpkgs", "utils": "utils" } }, "utils": { "locked": { - "lastModified": 1629481132, - "narHash": "sha256-JHgasjPR0/J1J3DRm4KxM4zTyAj4IOJY8vIl75v/kPI=", + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", "owner": "numtide", "repo": "flake-utils", - "rev": "997f7efcb746a9c140ce1f13c72263189225f482", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index b52bc7e..567a3dc 100644 --- a/flake.nix +++ b/flake.nix @@ -1,96 +1,28 @@ { - description = "Userspace tools for bcachefs"; - - # Nixpkgs / NixOS version to use. - inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; - inputs.utils.url = "github:numtide/flake-utils"; - inputs.filter.url = "github:numtide/nix-filter"; - - outputs = { self, nixpkgs, utils, filter, ... }@inputs: - let - # System types to support. - supportedSystems = [ "x86_64-linux" ]; - in - { - version = "${builtins.substring 0 8 self.lastModifiedDate}-${self.shortRev or "dirty"}"; - - overlay = import ./nix/overlay.nix inputs; - nixosModule = self.nixosModules.bcachefs; - nixosModules.bcachefs = import ./rust-src/mount/module.nix; - nixosModules.bcachefs-enable-boot = ({config, pkgs, lib, ... }:{ - # Disable Upstream NixOS Module when this is in use - disabledModules = [ "tasks/filesystems/bcachefs.nix" ]; - # Import needed packages - nixpkgs.overlays = [ self.overlay ]; - - # Add bcachefs to boot and kernel - boot.initrd.supportedFilesystems = [ "bcachefs" ]; - boot.supportedFilesystems = [ "bcachefs" ]; - }); - - nixosConfigurations.netboot-bcachefs = self.systems.netboot-bcachefs "x86_64-linux"; - systems.netboot-bcachefs = system: (nixpkgs.lib.nixosSystem { - inherit system; modules = [ - self.nixosModule - self.nixosModules.bcachefs-enable-boot - ("${nixpkgs}/nixos/modules/installer/netboot/netboot-minimal.nix") - ({ lib, pkgs, config, ... }: { - # installation disk autologin - services.getty.autologinUser = lib.mkForce "root"; - users.users.root.initialPassword = "toor"; - - # Symlink everything together - system.build.netboot = pkgs.symlinkJoin { - name = "netboot"; - paths = with config.system.build; [ - netbootRamdisk - kernel - netbootIpxeScript - ]; - preferLocalBuild = true; - }; - }) - ]; - }); - } - // utils.lib.eachSystem supportedSystems (system: - let pkgs = import nixpkgs { - inherit system; - overlays = [ self.overlay ]; - }; - in rec { - - # A Nixpkgs overlay. - - # Provide some binary packages for selected system types. - defaultPackage = pkgs.bcachefs.tools; - packages = { - inherit (pkgs.bcachefs) - tools - toolsValgrind - toolsDebug - mount - bch_bindgen - kernel; - - tools-musl = pkgs.pkgsMusl.bcachefs.tools; - mount-musl = pkgs.pkgsMusl.bcachefs.mount; - }; - - checks = { - kernelSrc = packages.kernel.src; - inherit (packages) - mount - bch_bindgen - toolsValgrind; - - # Build and test initrd with bcachefs and bcachefs.mount installed - # Disabled Test because it takes a while to build the kernel - # bootStage1Module = self.nixosConfigurations.netboot-bcachefs.config.system.build.bootStage1; - }; - - devShell = devShells.tools; - devShells.tools = pkgs.bcachefs.tools.override { inShell = true; }; - devShells.mount = pkgs.bcachefs.mount.override { inShell = true; }; - }); + description = "Userspace tools for bcachefs"; + + # Nixpkgs / NixOS version to use. + inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + inputs.utils.url = "github:numtide/flake-utils"; + inputs.flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + + outputs = { self, nixpkgs, utils, ... }: + utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + binary = pkgs.callPackage ./binary.nix { }; + mount = pkgs.callPackage ./rust-src/mount/default.nix { inherit binary; }; + bcachefs = pkgs.callPackage ./base.nix { + inherit binary mount; + testWithValgrind = false; + }; + in { + packages = { + inherit binary mount; + default = bcachefs; + }; + }); } diff --git a/nix/bcachefs-kernel.nix b/nix/bcachefs-kernel.nix deleted file mode 100644 index c937df4..0000000 --- a/nix/bcachefs-kernel.nix +++ /dev/null @@ -1,34 +0,0 @@ -{ lib -, fetchpatch -, fetchgit -, fetchFromGitHub -, buildLinux -, commit -, sha256 ? lib.fakeSha256 -, kernelVersion ? "5.13.0" -, kernelPatches ? [] # must always be defined in bcachefs' all-packages.nix entry because it's also a top-level attribute supplied by callPackage -, argsOverride ? {} -, versionString ? (builtins.substring 0 8 commit) -, ... -} @ args: - -buildLinux { - inherit kernelPatches; - - # pname = "linux"; - version = "${kernelVersion}-bcachefs-${versionString}"; - - modDirVersion = kernelVersion; - - - src = fetchFromGitHub { - name = "bcachefs-kernel-src"; - owner = "koverstreet"; - repo = "bcachefs"; - rev = commit; - inherit sha256; - }; - - extraConfig = "BCACHEFS_FS m"; - # NIX_DEBUG=5; -} \ No newline at end of file diff --git a/nix/bcachefs.rev.sha256 b/nix/bcachefs.rev.sha256 deleted file mode 100644 index 3f06215..0000000 --- a/nix/bcachefs.rev.sha256 +++ /dev/null @@ -1 +0,0 @@ -sha256-JsWrbuxrs047YKGES+r7mMfPdDWIMAGrg1fWi8qU4+A= \ No newline at end of file diff --git a/nix/fetchnix.nix b/nix/fetchnix.nix deleted file mode 100644 index 2f98788..0000000 --- a/nix/fetchnix.nix +++ /dev/null @@ -1,48 +0,0 @@ -# `builtins.fetchTarball` only accepts a `sha256` argument in Nix version 1.12 -# or later, so here we provide a function that can provide a compatible interface -# to Nix 1.11 or Nix 1.12 -# -# TODO FIXME: remove this sometime after Nix 1.12 goes stable - -{ url # URL of the nixpkgs tarball to download -, rev # The Git revision of nixpkgs to fetch -, sha256 # The SHA256 of the downloaded data -, system ? builtins.currentSystem # This is overridable if necessary -}: - -with { - ifThenElse = { bool, thenValue, elseValue }: ( - if bool then thenValue else elseValue); -}; - -ifThenElse { - bool = (0 <= builtins.compareVersions builtins.nixVersion "1.12"); - - # In Nix 1.12, we can just give a `sha256` to `builtins.fetchTarball`. - thenValue = (builtins.fetchTarball { inherit url sha256; }); - - # This hack should at least work for Nix 1.11 - elseValue = ( - (rec { - tarball = import { inherit url sha256; }; - builtin-paths = import ; - - script = builtins.toFile "nixpkgs-unpacker" '' - "$coreutils/mkdir" "$out" - cd "$out" - "$gzip" --decompress < "$tarball" | "$tar" -x --strip-components=1 - ''; - - nixpkgs = builtins.derivation { - name = "nixpkgs-${builtins.substring 0 6 rev}"; - - builder = builtins.storePath builtin-paths.shell; - args = [ script ]; - - inherit tarball system; - tar = builtins.storePath builtin-paths.tar; - gzip = builtins.storePath builtin-paths.gzip; - coreutils = builtins.storePath builtin-paths.coreutils; - }; - }).nixpkgs); -} diff --git a/nix/nixpkgs.json b/nix/nixpkgs.json deleted file mode 100644 index a5a11d0..0000000 --- a/nix/nixpkgs.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "url": "https://github.com/nixos/nixpkgs/archive/5ae883b8c3b04e0c4a9c92a5ab3c7c84b9942943.tar.gz", - "rev": "5ae883b8c3b04e0c4a9c92a5ab3c7c84b9942943", - "sha256": "1s2nhax586v2fax7r5qd1s3d2gdg25isva7k7r9pf9x9ay630cmb" -} diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix deleted file mode 100644 index 0067366..0000000 --- a/nix/nixpkgs.nix +++ /dev/null @@ -1,9 +0,0 @@ -let - # Grab the versions we specified in the JSON file - nixpkgs = builtins.fromJSON (builtins.readFile ./nixpkgs.json); - - # Bootstrap a copy of nixpkgs, based on this. - src = import ./fetchnix.nix { inherit (nixpkgs) url rev sha256; }; - -# We use the default nixpkgs configuration during bootstrap. -in import src { config = {}; } diff --git a/nix/overlay.nix b/nix/overlay.nix deleted file mode 100644 index 8138f20..0000000 --- a/nix/overlay.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ filter, self, ... }: -final: prev: { - bcachefs = { - tools = final.callPackage ../default.nix { - testWithValgrind = false; - filter = filter.lib; - versionString = self.version; - }; - toolsValgrind = final.bcachefs.tools.override { - testWithValgrind = true; - }; - toolsDebug = final.bcachefs.toolsValgrind.override { - debugMode = true; - }; - - bch_bindgen = final.callPackage ../rust-src/bch_bindgen {}; - - mount = final.callPackage ../rust-src/mount {}; - - kernelPackages = final.recurseIntoAttrs (final.linuxPackagesFor final.bcachefs.kernel); - kernel = final.callPackage ./bcachefs-kernel.nix { - commit = final.bcachefs.tools.bcachefs_revision; - # This needs to be recalculated for every revision change - sha256 = builtins.readFile ./bcachefs.rev.sha256; - kernelPatches = []; - }; - }; -} diff --git a/nix/update-nixpkgs.sh b/nix/update-nixpkgs.sh deleted file mode 100755 index 770d280..0000000 --- a/nix/update-nixpkgs.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -e - -if [[ "x$1" == "x" ]]; then - echo "Must provide a revision argument" - echo "Usage:" - echo " ./update-nixpkgs.sh " - echo " ./update-nixpkgs.sh https://github.com/foo/nixpkgs " - exit 1 -fi - -if [[ "x$2" == "x" ]]; then - REV="$1" - URL="https://github.com/nixos/nixpkgs" -else - REV="$2" - URL="$1" -fi - -DOWNLOAD="$URL/archive/$REV.tar.gz" -echo "Updating to nixpkgs revision $REV from $URL" -SHA256=$(nix-prefetch-url "$DOWNLOAD") - -cat > nixpkgs.json <