bpi-r3: more work on booting

(still doesnt boot)

also some misc changes not related to bpi-r3

default.nix

@@ -0,0 +1,10 @@
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
+    fetchTarball {
+      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ./.; }
+).defaultNix

flake.lock

@@ -17,6 +17,22 @@
       }
     },
     "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
       "flake": false,
       "locked": {
         "lastModified": 1668681692,
@@ -142,7 +158,7 @@
     "nixos-mailserver": {
       "inputs": {
         "blobs": "blobs",
-        "flake-compat": "flake-compat",
+        "flake-compat": "flake-compat_2",
         "nixpkgs": [
           "nixpkgs"
         ],
@@ -237,6 +253,7 @@
     },
     "root": {
       "inputs": {
+        "flake-compat": "flake-compat",
         "home-manager": "home-manager",
         "impermanence": "impermanence",
         "nix-gaming": "nix-gaming",

flake.nix

@@ -27,6 +27,10 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.nixpkgs-22_11.follows = "nixpkgs";
     };
+    flake-compat = {
+      url = "github:edolstra/flake-compat";
+      flake = false;
+    };
   };
 
   outputs = inputs@{ self, nixpkgs, nixos-hardware, impermanence, home-manager, nur, nix-gaming, notlua, nixos-mailserver, ... }:

home/common/waybar.nix

@@ -18,7 +18,14 @@
       # position = "bottom";
       ipc = true;
       height = 40;
-      modules-left = [ "sway/workspaces" "sway/mode" "mpris" ];
+      modules-left = [ "sway/workspaces" "sway/mode" "idle_inhibitor" "mpris" ];
+      idle_inhibitor = {
+        format = "{icon}";
+        format-icons = {
+          activated = "󰅶";
+          deactivated = "󰾪";
+        };
+      };
       mpris = {
         tooltip = true;
         format = "{player_icon} {dynamic}";

home/hosts/nixmsi.nix

@@ -106,6 +106,7 @@
     glaxnimate
     lalrpop
     looking-glass-client
+    tio
   ];
   xdg.configFile."looking-glass/client.ini".text = ''
     [app]

pkgs/_sources/generated.json

@@ -23,11 +23,11 @@
         "pinned": false,
         "src": {
             "name": null,
-            "sha256": "sha256-JYGwb0LhIs6B2/OHiU+mJ/dAAS+Dg+MrVksAsn6IS9g=",
+            "sha256": "sha256-OPwmVxBGaWo51pDJcqvxvZ8qxMH8X0DwZTpwiKbdx/I=",
             "type": "url",
-            "url": "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-3/GE-Proton8-3.tar.gz"
+            "url": "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-4/GE-Proton8-4.tar.gz"
         },
-        "version": "GE-Proton8-3"
+        "version": "GE-Proton8-4"
     },
     "yomichan": {
         "cargoLocks": null,

pkgs/_sources/generated.nix

@@ -11,10 +11,10 @@
   };
   proton-ge = {
     pname = "proton-ge";
-    version = "GE-Proton8-3";
+    version = "GE-Proton8-4";
     src = fetchurl {
-      url = "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-3/GE-Proton8-3.tar.gz";
+      url = "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-4/GE-Proton8-4.tar.gz";
-      sha256 = "sha256-JYGwb0LhIs6B2/OHiU+mJ/dAAS+Dg+MrVksAsn6IS9g=";
+      sha256 = "sha256-OPwmVxBGaWo51pDJcqvxvZ8qxMH8X0DwZTpwiKbdx/I=";
     };
   };
   yomichan = {

pkgs/bpi-r3-atf-backport-mkimage-support.patch

@@ -0,0 +1,31 @@
+diff --git a/plat/mediatek/mt7986/platform.mk b/plat/mediatek/mt7986/platform.mk
+index c232d7a..27a165d 100644
+--- a/plat/mediatek/mt7986/platform.mk
++++ b/plat/mediatek/mt7986/platform.mk
+@@ -456,6 +456,7 @@ else
+ bl2: $(BUILD_PLAT)/bl2.img
+ endif
+ 
++ifneq ($(USE_MKIMAGE),1)
+ ifneq ($(BROM_SIGN_KEY),)
+ $(BUILD_PLAT)/bl2.img: $(BROM_SIGN_KEY)
+ endif
+@@ -472,6 +473,18 @@ $(BUILD_PLAT)/bl2.img: $(BL2_IMG_PAYLOAD) $(DOIMAGETOOL)
+ 		$(if $(BL_AR_VER), -r $(BL_AR_VER))				\
+ 		$(if $(NAND_TYPE), -n $(NAND_TYPE))				\
+ 		$(BL2_IMG_PAYLOAD) $@
++else
++MKIMAGE ?= mkimage
++
++ifneq ($(BROM_SIGN_KEY)$(AUTO_AR_VER),)
++$(warning BL2 signing/anti-rollback is not supported using mkimage)
++endif
++
++$(BUILD_PLAT)/bl2.img: $(BL2_IMG_PAYLOAD)
++	$(Q)$(MKIMAGE) -T mtk_image -a $(BL2_BASE) -e $(BL2_BASE)		\
++		-n "arm64=1;media=$(BROM_HEADER_TYPE)$(if $(NAND_TYPE),;nandinfo=$(NAND_TYPE))"	\
++		-d $(BL2_IMG_PAYLOAD) $@
++endif
+ 
+ $(DOIMAGETOOL):
+ 	$(Q)$(MAKE) --no-print-directory -C $(DOIMAGEPATH)

pkgs/default.nix

@@ -8,12 +8,73 @@ let
   sources = import ./_sources/generated.nix {
     inherit (pkgs) fetchgit fetchurl fetchFromGitHub dockerTools;
   };
+  armTrustedFirmwareBpiR3 = { bootDevice, uboot ? null }: pkgs.buildArmTrustedFirmware rec {
+    # TODO: nvfetcherify this
+    src = pkgs.fetchFromGitHub {
+      owner = "frank-w";
+      repo = "u-boot";
+      rev = "c30a1caf8274af67bf31f3fb5abc45df5737df36";
+      hash = "sha256-pW2yytXRIFEIbG1gnuXq8TiLe/Eew7zESe6Pijh2qVk=";
+    };
+    patches = [ ./bpi-r3-atf-backport-mkimage-support.patch ];
+    extraMakeFlags = assert builtins.elem bootDevice [
+      "nor" "snand" "spim-nand" "emmc" "sdmmc" "ram"
+    ]; [
+      "BOOT_DEVICE=${bootDevice}"
+      "DRAM_USE_DDR4=1"
+      "USE_MKIMAGE=1"
+      "MKIMAGE=${pkgs.ubootTools}/bin/mkimage"
+      "all"
+      "fip"
+    ] ++ lib.optionals (uboot != null) [
+      "BL33=${uboot}/u-boot.bin"
+    ];
+    extraMeta.platforms = [ "aarch64-linux" ];
+    platform = "mt7986";
+    filesToInstall = [
+      "build/${platform}/release/bl2.img"
+      "build/${platform}/release/fip.bin"
+    ];
+    nativeBuildInputs = with pkgs; [ /*pkgsCross.arm-embedded.stdenv.cc*/ dtc ];
+  };
+  # sd/emmc
+  # -- CONFIG_USE_BOOTCOMMAND/CONFIG_BOOTCOMMAND - distroboot stuff (override default boot command)
+  # -- CONFIG_BOOTDELAY - autoboot timeout
+  # CONFIG_BOOTSTD_DEFAULTS - stdboot stuff
+  # CONFIG_BOOTSTD_BOOTCOMMAND - might be? an alternative to CONFIG_BOOTCOMMAND
+  # CONFIG_DEFAULT_FDT_FILE - compatibility with nixos
+  # CONFIG_DISTRO_DEFAULTS - surely this won't hurt, it adds autocomplete and stuff and doesn't weight much in the large scale of things
+  # CONFIG_SYS_BOOTM_LEN - increase max initrd? size
+  # CONFIG_ZSTD - allow zstd initrd
+  ubootConfig = storage: ''
+    CONFIG_AUTOBOOT=y
+    CONFIG_BOOTCOMMAND="${builtins.replaceStrings [ "\n" ] [ "; " ] ''
+        setenv boot_prefixes /@boot/ /@/ /boot/ /
+        run distro_bootcmd
+    ''};"
+    CONFIG_BOOTSTD_DEFAULTS=y
+    CONFIG_BOOTSTD_FULL=y
+    CONFIG_CMD_BTRFS=y
+    CONFIG_CMD_CAT=y
+    CONFIG_DEFAULT_FDT_FILE="mediatek/mt7986a-bananapi-bpi-r3.dtb"
+    CONFIG_DISTRO_DEFAULTS=y
+    CONFIG_ENV_IS_NOWHERE=y
+    CONFIG_FS_BTRFS=y
+    CONFIG_SYS_BOOTM_LEN=0x6000000
+    CONFIG_USE_BOOTCOMMAND=y
+    CONFIG_ZSTD=y
+  '';
+  ubootVersion = "2023.07-rc3";
+  ubootSrc = pkgs.fetchurl {
+    url = "ftp://ftp.denx.de/pub/u-boot/u-boot-${ubootVersion}.tar.bz2";
+    hash = "sha256-QuwINnS9MPpMFueMP19FPAjZ9zdZWne13aWVrDoJ2C8=";
+  };
 in
 
-{
+rec {
   osu-lazer-bin = nix-gaming.osu-lazer-bin;
-  clang-tools_latest = pkgs.clang-tools_15;
+  clang-tools_latest = pkgs.clang-tools_16;
-  clang_latest = pkgs.clang_15;
+  clang_latest = pkgs.clang_16;
   home-daemon = callPackage ./home-daemon { };
   /*ghidra = pkgs.ghidra.overrideAttrs (old: {
     patches = old.patches ++ [ ./ghidra-stdcall.patch ];
@@ -43,6 +104,81 @@ in
   # system76-scheduler = callPackage ./system76-scheduler.nix { };
   techmino = callPackage ./techmino { };
 
+  ubootBpiR3Sd = pkgs.buildUBoot {
+    defconfig = "mt7986a_bpir3_sd_defconfig";
+    extraConfig = ubootConfig "sd";
+    src = ubootSrc;
+    version = ubootVersion;
+    extraMeta.platforms = [ "aarch64-linux" ];
+    # https://github.com/nakato/nixos-bpir3-example/blob/main/pkgs/uboot/mt7986-default-bootcmd.patch
+    patches = [ ./mt7986-default-bootcmd.patch ];
+    filesToInstall = [ "u-boot.bin" ];
+  };
+  ubootBpiR3Emmc = pkgs.buildUBoot {
+    defconfig = "mt7986a_bpir3_emmc_defconfig";
+    extraConfig = ubootConfig "emmc";
+    src = ubootSrc;
+    version = ubootVersion;
+    extraMeta.platforms = [ "aarch64-linux" ];
+    patches = [ ./mt7986-default-bootcmd.patch ];
+    filesToInstall = [ "u-boot.bin" ];
+  };
+  armTrustedFirmwareBpiR3Sd = armTrustedFirmwareBpiR3 { uboot = ubootBpiR3Sd; bootDevice = "sdmmc"; };
+  armTrustedFirmwareBpiR3Emmc = armTrustedFirmwareBpiR3 { uboot = ubootBpiR3Emmc; bootDevice = "emmc"; };
+  bpiR3StuffCombined = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out/sd
+      mkdir -p $out/emmc
+      cp ${bpiR3StuffEmmc}/* $out/emmc
+      cp ${bpiR3StuffSd}/* $out/sd
+    '';
+    fixupPhase = "true";
+  };
+  bpiR3StuffEmmc = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff-emmc";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out
+      cp ${ubootBpiR3Emmc}/*.* $out
+      cp ${armTrustedFirmwareBpiR3Emmc}/*.* $out
+    '';
+    fixupPhase = "true";
+  };
+  bpiR3StuffSd = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff-sd";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out
+      cp ${ubootBpiR3Sd}/*.* $out
+      cp ${armTrustedFirmwareBpiR3Sd}/*.* $out
+    '';
+    fixupPhase = "true";
+  };
+  linux_bpiR3 = (pkgs.buildLinux ({
+    version = "6.3";
+    modDirVersion = "6.3.0";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "frank-w";
+      repo = "BPI-Router-Linux";
+      rev = "6.3-main";
+      hash = lib.fakeHash;
+    };
+
+    defconfig = "mt7986a_bpi-r3";
+  })).overrideAttrs (old: {
+    postConfigure = ''
+      sed -i "$buildRoot/.config" -e 's%^CONFIG_LOCALVERSION=.*%CONFIG_LOCALVERSION=""%'
+      sed -i "$buildRoot/include/config/auto.conf" -e 's%^CONFIG_LOCALVERSION=.*%CONFIG_LOCALVERSION=""%'
+    '';
+  });
+  linuxPackages_bpiR3 = pkgs.linuxPackagesFor linux_bpiR3;
+
   firefox-addons = lib.recurseIntoAttrs (callPackage ./firefox-addons { inherit nur sources; });
   mpvScripts = pkgs.mpvScripts // (callPackage ./mpv-scripts { });
 }

pkgs/mt7986-default-bootcmd.patch

@@ -0,0 +1,75 @@
+diff --git a/include/configs/mt7986.h b/include/configs/mt7986.h
+index 0c41af1fc3..4ddf3cef89 100644
+--- a/include/configs/mt7986.h
++++ b/include/configs/mt7986.h
+@@ -9,6 +9,8 @@
+ #ifndef __MT7986_H
+ #define __MT7986_H
+ 
++#include <linux/stringify.h>
++
+ /* Uboot definition */
+ #define CFG_SYS_UBOOT_BASE		CONFIG_TEXT_BASE
+ 
+@@ -18,4 +20,61 @@
+ /* DRAM */
+ #define CFG_SYS_SDRAM_BASE		0x40000000
+ 
++#define BOOT_TARGET_DEVICES(func) \
++	func(MMC, mmc, 0)
++
++#include <config_distro_bootcmd.h>
++
++/*
++ * Linux mt6978a.dtsi defines reserved memory in the first 256MB:
++ * 0x4300_0000 - 0x4302_FFFF
++ * 0x4FC0_0000 - 0x4FCF_FFFF
++ * 0x4FD0_0000 - 0x4FD3_FFFF
++ * 0x4FD4_0000 - 0x4FD7_FFFF
++ * 0x4FD8_0000 - 0x4FDA_3FFF
++ *
++ * Getting uboot to obey these by adding them into the DTS here
++ * got it respecting them.  To not clobber these memory addresses
++ * fdt_high and initrd_high are being set.
++ */
++
++/*
++ * Do not relocate FDT, because relocate fails to alloc when
++ * initrd_high is set to 0x4FC0_0000 to keep initrd out of
++ * reserved memory.  This is irregardless of the reserved
++ * addresses above still being valid as far as configuraiton
++ * is concerned at that point, so I'm at a loss.
++ */
++#define	FDT_HIGH	__stringify(0xffffffffffffffff)
++/* No reloc might work now that FDT is in-place, but this should be more compatible */
++#define	INITRD_HIGH	__stringify(0x4fc00000)
++
++/* Seems to be the address uboot would relocate to so place there */
++#define	KERNEL_ADDR_R	__stringify(0x43200000)
++/* Right before the kernel, after that reserved memory, probably best spots exist */
++#define	FDT_ADDR_R	__stringify(0x43030000)
++
++#define	RAMDISK_ADDR_R	__stringify(0x4b100000)
++
++/* Anything different means no-boot */
++#define	BOOTM_SIZE	__stringify(0x10000000)
++
++/* These can be outside Linux's 256MB */
++#define	SCRIPTADDR	__stringify(0x90000000)
++#define	PXEFILE_ADDR_R	__stringify(0x90100000)
++
++
++/* No comp_addr supplied */
++#define CFG_EXTRA_ENV_SETTINGS \
++	"bootm_size=" BOOTM_SIZE "\0" \
++	"fdt_addr_r=" FDT_ADDR_R "\0" \
++	"fdt_high=" FDT_HIGH "\0" \
++	"initrd_high=" INITRD_HIGH "\0" \
++	"kernel_addr_r=" KERNEL_ADDR_R "\0" \
++	"pxefile_addr_r=" PXEFILE_ADDR_R "\0" \
++	"ramdisk_addr_r=" RAMDISK_ADDR_R "\0" \
++	"scriptaddr=" SCRIPTADDR "\0" \
++	"fdtfile=" CONFIG_DEFAULT_FDT_FILE "\0" \
++	BOOTENV
++
+ #endif

pkgs/nvfetcher.toml

@@ -10,3 +10,13 @@ fetch.url = "https://github.com/FooSoft/yomichan/releases/download/$ver/yomichan
 # allow prereleases
 src.github_tag = "FastForwardTeam/FastForward"
 fetch.url = "https://github.com/FastForwardTeam/FastForward/releases/download/$ver/fastforwardteam-$ver.xpi"
+
+# nix-prefetch doesnt work with git right now for some reason, whatever
+# [atf-bpir3]
+# src.git = "https://github.com/frank-w/u-boot.git"
+# src.branch = "r3-atf"
+# src.use_commit = true
+# fetch.git = "https://github.com/frank-w/u-boot.git"
+# fetch.branch = "$ver"
+#fetch.github = "frank-w/u-boot"
+# fetch.branch = "r3-atf"

system/hardware/bpi_r3.nix

@@ -1,35 +1,26 @@
 { pkgs
-, lib
 , ... }:
 
-# WIP
 {
   boot.loader = {
     grub.enable = false;
     generic-extlinux-compatible.enable = true;
   };
 
-  # https://github.com/frank-w/BPI-Router-Linux
+  # i'm not about to build a kernel on every update without an arm device...
-  boot.kernelPackages = pkgs.linuxPackagesFor ((pkgs.buildLinux ({
+  # i guess i could use my phone for building it, but no, not interested
-    version = "6.3";
+  boot.kernelPackages = pkgs.linuxPackages_testing;
-    modDirVersion = "6.3.0";
+  # boot.kernelPackages = pkgs.linuxPackages_bpiR3;
-
-    src = pkgs.fetchFromGitHub {
-      owner = "frank-w";
-      repo = "BPI-Router-Linux";
-      rev = "6.3-main";
-      hash = lib.fakeHash;
-    };
-
-    defconfig = "mt7986a_bpi-r3";
-  })).overrideAttrs (old: {
-    postConfigure = ''
-      sed -i "$buildRoot/.config" -e 's/^CONFIG_LOCALVERSION=.*/CONFIG_LOCALVERSION=""/'
-      sed -i "$buildRoot/include/config/auto.conf" -e 's/^CONFIG_LOCALVERSION=.*/CONFIG_LOCALVERSION=""/'
-    '';
-  }));
 
   hardware.deviceTree.enable = true;
-  hardware.deviceTree.filter = "mt7986a-bananapi-bpi-r3*.dtb";
+  hardware.deviceTree.filter = "*mt7986*";
   hardware.enableRedistributableFirmware = true;
+
+  # # disable a bunch of useless drivers
+  # boot.initrd.includeDefaultModules = false;
+  # boot.initrd.availableKernelModules = [ "mmc_block" "dm_mod" "rfkill" "cfg80211" "mt7915e" "ubi" "sdhci" "sd_mod" "sr_mod" "xhci_pci" ];
+  boot.kernelParams = [ "console=ttyS0,115200" ];
+
+  boot.initrd.compressor = "zstd";
+  nixpkgs.buildPlatform = "x86_64-linux";
 }

system/hosts/nixmsi.nix

@@ -231,5 +231,7 @@ in {
   };
 
   programs.ccache.enable = true;
+  services.sshd.enable = true;
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 }
 

system/hosts/router/default.nix

@@ -1,9 +1,17 @@
 { config
+, pkgs
 , ... }:
 
+# EMMC size: 7818182656
+
 let
-  rootUuid = "00000000-0000-0000-0000-000000000000";
+  rootUuid = "44444444-4444-4444-8888-888888888888";
   rootPart = "/dev/disk/by-uuid/${rootUuid}";
+  rootfsImage = pkgs.callPackage "${pkgs.path}/nixos/lib/make-ext4-fs.nix" {
+    storePaths = config.system.build.toplevel;
+    compressImage = false;
+    volumeLabel = "NIX_ROOTFS";
+  };
 in {
   system.stateVersion = "22.11";
   # TODO
@@ -13,16 +21,41 @@ in {
                 options = [ "defaults" "size=2G" "mode=755" ]; };
     "/persist" =
               { device = rootPart; fsType = "btrfs"; neededForBoot = true;
-                options = [ ]; };
+                options = [ "compress=zstd:15" "subvol=@" ]; };
+    "/boot" =
+              { device = rootPart; fsType = "btrfs"; neededForBoot = true;
+                options = [ "subvol=@boot" ]; };
+    "/nix" =
+              { device = rootPart; fsType = "btrfs"; neededForBoot = true;
+                options = [ "compress=zstd:15" "subvol=@nix" ]; };
   };
+  services.openssh.enable = true;
   impermanence = {
     enable = true;
     path = /persist;
     directories = [
       { directory = /home/${config.common.mainUsername}; user = config.common.mainUsername; group = config.common.mainUsername; mode = "0700"; }
       { directory = /root; mode = "0700"; }
-      /nix
-      /boot
     ];
   };
+  hardware.wirelessRegulatoryDatabase = true;
+  system.build.emmcImage = pkgs.callPackage ./image.nix {
+    inherit config rootfsImage;
+    bpiR3Stuff = pkgs.bpiR3StuffEmmc;
+  };
+  system.build.sdImage = pkgs.callPackage ./image.nix {
+    inherit config rootfsImage;
+    bpiR3Stuff = pkgs.bpiR3StuffSd;
+  };
+  system.build.rootfs = rootfsImage;
+  boot.postBootCommands = ''
+    if [ -f ${toString config.impermanence.path}/nix-path-registration ]; then
+      ${config.nix.package.out}/bin/nix-store --load-db < ${toString config.impermanence.path}/nix-path-registration
+      mkdir -p /etc
+      touch /etc/NIXOS
+      ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+      rm -f ${toString config.impermanence.path}/nix-path-registration
+    fi
+  '';
+  boot.kernelParams = [ "boot.shell_on_fail" ];
 }

system/hosts/router/image.nix

@@ -0,0 +1,99 @@
+{ gptfdisk
+, e2fsprogs
+, util-linux
+, zstd
+, btrfs-progs
+, vmTools
+, runCommand
+, jq
+, kmod
+, udev
+, nix
+, lkl
+
+, bpiR3Stuff
+, config
+, rootfsImage
+, ... }:
+
+let
+  imageSize = 7818182656;
+  bl2Start = 34;
+  bl2End = 8191;
+  envStart = 8192;
+  envEnd = 9215;
+  factoryStart = 9216;
+  factoryEnd = 13311;
+  fipStart = 13312;
+  fipEnd = 17407;
+  rootPartStart = fipEnd + 1;
+  rootPartEnd = builtins.floor (imageSize / 512) - 100;
+in
+
+# the vm is suuuuuper slow
+# so, do as much as possible outside of it
+# but i still need it to create subvolumes
+
+vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
+  enableParallelBuildingByDefault = true;
+  nativeBuildInputs = [
+    btrfs-progs
+    # dosfstools
+    e2fsprogs
+    util-linux # sfdisk
+    config.system.build.nixos-install
+    jq
+    kmod
+    udev
+    gptfdisk
+    zstd
+    nix
+    lkl
+  ];
+  preVM = ''
+    img=./result.img
+    ls /dev/fuse /etc/mtab || (echo "Must have /dev/fuse and /etc/mtab in nix options (extra-sandbox-paths)" && exit 1)
+    ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
+    truncate -s ${toString imageSize} $img
+
+    ${gptfdisk}/bin/sgdisk -o \
+      --set-alignment=2 \
+      -n 1:${toString bl2Start}:${toString bl2End} -c 1:bl2 -A 1:set:2 \
+      -n 2:${toString envStart}:${toString envEnd} -c 2:u-boot-env \
+      -n 3:${toString factoryStart}:${toString factoryEnd} -c 3:factory \
+      -n 4:${toString fipStart}:${toString fipEnd} -c 4:fip \
+      -n 5:${toString rootPartStart}:${toString rootPartEnd} -c 5:root -A 5:set:2 \
+      $img
+    dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
+    dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
+    truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
+    ${btrfs-progs}/bin/mkfs.btrfs \
+      --label NIXOS_SD \
+      --uuid "44444444-4444-4444-8888-888888888889" \
+      ./tmp.img
+  '';
+  # i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
+  # (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
+  postVM = ''
+    # Truncate the file at the end of the last partition
+    img=./result.img
+    mkdir -p $out/boot
+    ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
+    cp ${rootfsImage} $out/rootfs.ext4
+    cp tmp.img $out/template.btrfs
+    cp $img $out/image.img
+    echo ${toString rootPartStart} > $out/metadata
+    echo ${toString rootPartEnd} >> $out/metadata
+  '';
+  memSize = "4G";
+  QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
+} ''
+  modprobe btrfs
+  mkdir -p /mnt
+  mount -t btrfs -o space_cache=v2 /dev/vda /mnt
+  btrfs filesystem resize max /mnt
+  btrfs subvolume create /mnt/@boot
+  btrfs subvolume create /mnt/@
+  btrfs subvolume create /mnt/@nix
+  chattr +C /mnt/@boot
+'')

system/hosts/router/image.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# this step requires either root access or access to fuse
+# nix builder doesnt have access to either
+# i could run this in a vm, but that's slow as fuck because i'm building this on x86_64-linux
+# usage: image.sh [path to image.nix output]
+
+set -euxo pipefail
+
+(which zstd && which rsync) || exit 1
+
+userspace="$(which lklfuse >/dev/null && echo -n 1 || echo -n)"
+use_rsync="$(which rsync >/dev/null && echo -n 1 || echo -n)"
+function Mount() {
+  if [[ "$userspace" == "1" ]]; then
+    if [[ "${4-}" == "ro" ]]; then
+      lklfuse -o "type=$1" -o "$4" "$2" "$3"
+    elif [ -n "${4-}" ]; then
+      lklfuse -o "type=$1" -o "opts=$4" "$2" "$3"
+    else
+      lklfuse -o "type=$1" "$2" "$3"
+    fi
+  elif [ -n "${4-}" ]; then
+    sudo -A mount -t "$1" -o "loop,$4" "$2" "$3"
+  else
+    sudo -A mount -t "$1" -o loop "$2" "$3"
+  fi
+}
+function run() {
+  if [[ "$userspace" == "1" ]]; then
+    "$@"
+  else
+    sudo -A "$@"
+  fi
+}
+function cpr() {
+  if [[ "$use_rsync" == "1" ]]; then
+    run rsync -a --info=progress2 "$1/" "$2/"
+  else
+    run cp -rv "$1"/* "$2/"
+  fi
+}
+
+if [ -z "$1" ]; then
+  echo "missing argument"
+  exit 1
+fi
+rootfs="$1/rootfs.ext4"
+template="$1/template.btrfs"
+image="$1/image.img"
+boot="$1/boot"
+
+metadata0="$(head -n1 "$1/metadata")"
+metadata1="$(head -n2 "$1/metadata" | tail -n1)"
+if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1" ]]; then
+  echo "invalid metadata"
+  exit 1
+fi
+
+tmp=$(mktemp -d)
+cp "$template" "$tmp/template.btrfs"
+cp "$image" "$tmp/image.img"
+template="$tmp/template.btrfs"
+image="$tmp/image.img"
+chmod +w "$template" "$image"
+
+function cleanup {
+  run umount "$rootfs" || echo -n
+  run umount "$tmp/out" || echo -n
+  rm -rf "$tmp"
+}
+trap cleanup EXIT
+mkdir -p "$tmp/rootfs" "$tmp/out"
+Mount ext4 "$rootfs" "$tmp/rootfs" ro
+rootfs="$tmp/rootfs"
+Mount btrfs "$template" "$tmp/out"
+run cp -rv "$boot"/* "$tmp/out/@boot/"
+run umount "$tmp/out"
+Mount btrfs "$template" "$tmp/out" "compress=zstd:15"
+run cp -v "$rootfs/nix-path-registration" "$tmp/out/@/"
+run ls "$boot"
+cpr "$boot" "$tmp/out/@boot"
+cpr "$rootfs/nix" "$tmp/out/@nix"
+
+run umount "$rootfs"
+run umount "$tmp/out"
+
+dd conv=notrunc if="$template" of="$image" seek="$metadata0"
+
+zstd --compress "$image"
+cp "$image.zst" ./image.img.zst

system/modules/impermanence.nix

@@ -13,6 +13,7 @@ in {
         };
         path = mkOption {
           type = types.path;
+          default = if cfg.enable then throw "You must set path to persistent storage" else "";
           description = "Default path for persistence";
         };
         directories = mkOption {