bpi-r3: more work on booting

(still doesnt boot) also some misc changes not related to bpi-r3
2023-06-11 18:02:38 +07:00 · 2023-06-11 18:02:38 +07:00 · a96bd4bbcd
parent ebc7241725
commit a96bd4bbcd
17 changed files with 545 additions and 37 deletions
--- a/default.nix
+++ b/default.nix
@ -0,0 +1,10 @@
+(import
+  (
+    let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
+    fetchTarball {
+      url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
+      sha256 = lock.nodes.flake-compat.locked.narHash;
+    }
+  )
+  { src = ./.; }
+).defaultNix
--- a/flake.lock
+++ b/flake.lock
@ -17,6 +17,22 @@
      }
    },
    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
      "flake": false,
      "locked": {
        "lastModified": 1668681692,
@ -142,7 +158,7 @@
    "nixos-mailserver": {
      "inputs": {
        "blobs": "blobs",
-        "flake-compat": "flake-compat",
+        "flake-compat": "flake-compat_2",
        "nixpkgs": [
          "nixpkgs"
        ],
@ -237,6 +253,7 @@
    },
    "root": {
      "inputs": {
+        "flake-compat": "flake-compat",
        "home-manager": "home-manager",
        "impermanence": "impermanence",
        "nix-gaming": "nix-gaming",
--- a/flake.nix
+++ b/flake.nix
@ -27,6 +27,10 @@
      inputs.nixpkgs.follows = "nixpkgs";
      inputs.nixpkgs-22_11.follows = "nixpkgs";
    };
+    flake-compat = {
+      url = "github:edolstra/flake-compat";
+      flake = false;
+    };
  };

  outputs = inputs@{ self, nixpkgs, nixos-hardware, impermanence, home-manager, nur, nix-gaming, notlua, nixos-mailserver, ... }:
--- a/home/common/waybar.nix
+++ b/home/common/waybar.nix
@ -18,7 +18,14 @@
      # position = "bottom";
      ipc = true;
      height = 40;
-      modules-left = [ "sway/workspaces" "sway/mode" "mpris" ];
+      modules-left = [ "sway/workspaces" "sway/mode" "idle_inhibitor" "mpris" ];
+      idle_inhibitor = {
+        format = "{icon}";
+        format-icons = {
+          activated = "󰅶";
+          deactivated = "󰾪";
+        };
+      };
      mpris = {
        tooltip = true;
        format = "{player_icon} {dynamic}";
--- a/home/hosts/nixmsi.nix
+++ b/home/hosts/nixmsi.nix
@ -106,6 +106,7 @@
    glaxnimate
    lalrpop
    looking-glass-client
+    tio
  ];
  xdg.configFile."looking-glass/client.ini".text = ''
    [app]
--- a/pkgs/_sources/generated.json
+++ b/pkgs/_sources/generated.json
@ -23,11 +23,11 @@
        "pinned": false,
        "src": {
            "name": null,
-            "sha256": "sha256-JYGwb0LhIs6B2/OHiU+mJ/dAAS+Dg+MrVksAsn6IS9g=",
+            "sha256": "sha256-OPwmVxBGaWo51pDJcqvxvZ8qxMH8X0DwZTpwiKbdx/I=",
            "type": "url",
-            "url": "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-3/GE-Proton8-3.tar.gz"
+            "url": "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-4/GE-Proton8-4.tar.gz"
        },
-        "version": "GE-Proton8-3"
+        "version": "GE-Proton8-4"
    },
    "yomichan": {
        "cargoLocks": null,
--- a/pkgs/_sources/generated.nix
+++ b/pkgs/_sources/generated.nix
@ -11,10 +11,10 @@
  };
  proton-ge = {
    pname = "proton-ge";
-    version = "GE-Proton8-3";
+    version = "GE-Proton8-4";
    src = fetchurl {
-      url = "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-3/GE-Proton8-3.tar.gz";
-      sha256 = "sha256-JYGwb0LhIs6B2/OHiU+mJ/dAAS+Dg+MrVksAsn6IS9g=";
+      url = "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/GE-Proton8-4/GE-Proton8-4.tar.gz";
+      sha256 = "sha256-OPwmVxBGaWo51pDJcqvxvZ8qxMH8X0DwZTpwiKbdx/I=";
    };
  };
  yomichan = {
--- a/pkgs/bpi-r3-atf-backport-mkimage-support.patch
+++ b/pkgs/bpi-r3-atf-backport-mkimage-support.patch
@ -0,0 +1,31 @@
+diff --git a/plat/mediatek/mt7986/platform.mk b/plat/mediatek/mt7986/platform.mk
+index c232d7a..27a165d 100644
+--- a/plat/mediatek/mt7986/platform.mk
+++ b/plat/mediatek/mt7986/platform.mk
+@@ -456,6 +456,7 @@ else
+ bl2: $(BUILD_PLAT)/bl2.img
+ endif
+ 
+ifneq ($(USE_MKIMAGE),1)
+ ifneq ($(BROM_SIGN_KEY),)
+ $(BUILD_PLAT)/bl2.img: $(BROM_SIGN_KEY)
+ endif
+@@ -472,6 +473,18 @@ $(BUILD_PLAT)/bl2.img: $(BL2_IMG_PAYLOAD) $(DOIMAGETOOL)
+ 		$(if $(BL_AR_VER), -r $(BL_AR_VER))				\
+ 		$(if $(NAND_TYPE), -n $(NAND_TYPE))				\
+ 		$(BL2_IMG_PAYLOAD) $@
+else
+MKIMAGE ?= mkimage
+
+ifneq ($(BROM_SIGN_KEY)$(AUTO_AR_VER),)
+$(warning BL2 signing/anti-rollback is not supported using mkimage)
+endif
+
+$(BUILD_PLAT)/bl2.img: $(BL2_IMG_PAYLOAD)
+	$(Q)$(MKIMAGE) -T mtk_image -a $(BL2_BASE) -e $(BL2_BASE)		\
+		-n "arm64=1;media=$(BROM_HEADER_TYPE)$(if $(NAND_TYPE),;nandinfo=$(NAND_TYPE))"	\
+		-d $(BL2_IMG_PAYLOAD) $@
+endif
+ 
+ $(DOIMAGETOOL):
+ 	$(Q)$(MAKE) --no-print-directory -C $(DOIMAGEPATH)
--- a/pkgs/default.nix
+++ b/pkgs/default.nix
@ -8,12 +8,73 @@ let
  sources = import ./_sources/generated.nix {
    inherit (pkgs) fetchgit fetchurl fetchFromGitHub dockerTools;
  };
+  armTrustedFirmwareBpiR3 = { bootDevice, uboot ? null }: pkgs.buildArmTrustedFirmware rec {
+    # TODO: nvfetcherify this
+    src = pkgs.fetchFromGitHub {
+      owner = "frank-w";
+      repo = "u-boot";
+      rev = "c30a1caf8274af67bf31f3fb5abc45df5737df36";
+      hash = "sha256-pW2yytXRIFEIbG1gnuXq8TiLe/Eew7zESe6Pijh2qVk=";
+    };
+    patches = [ ./bpi-r3-atf-backport-mkimage-support.patch ];
+    extraMakeFlags = assert builtins.elem bootDevice [
+      "nor" "snand" "spim-nand" "emmc" "sdmmc" "ram"
+    ]; [
+      "BOOT_DEVICE=${bootDevice}"
+      "DRAM_USE_DDR4=1"
+      "USE_MKIMAGE=1"
+      "MKIMAGE=${pkgs.ubootTools}/bin/mkimage"
+      "all"
+      "fip"
+    ] ++ lib.optionals (uboot != null) [
+      "BL33=${uboot}/u-boot.bin"
+    ];
+    extraMeta.platforms = [ "aarch64-linux" ];
+    platform = "mt7986";
+    filesToInstall = [
+      "build/${platform}/release/bl2.img"
+      "build/${platform}/release/fip.bin"
+    ];
+    nativeBuildInputs = with pkgs; [ /*pkgsCross.arm-embedded.stdenv.cc*/ dtc ];
+  };
+  # sd/emmc
+  # -- CONFIG_USE_BOOTCOMMAND/CONFIG_BOOTCOMMAND - distroboot stuff (override default boot command)
+  # -- CONFIG_BOOTDELAY - autoboot timeout
+  # CONFIG_BOOTSTD_DEFAULTS - stdboot stuff
+  # CONFIG_BOOTSTD_BOOTCOMMAND - might be? an alternative to CONFIG_BOOTCOMMAND
+  # CONFIG_DEFAULT_FDT_FILE - compatibility with nixos
+  # CONFIG_DISTRO_DEFAULTS - surely this won't hurt, it adds autocomplete and stuff and doesn't weight much in the large scale of things
+  # CONFIG_SYS_BOOTM_LEN - increase max initrd? size
+  # CONFIG_ZSTD - allow zstd initrd
+  ubootConfig = storage: ''
+    CONFIG_AUTOBOOT=y
+    CONFIG_BOOTCOMMAND="${builtins.replaceStrings [ "\n" ] [ "; " ] ''
+        setenv boot_prefixes /@boot/ /@/ /boot/ /
+        run distro_bootcmd
+    ''};"
+    CONFIG_BOOTSTD_DEFAULTS=y
+    CONFIG_BOOTSTD_FULL=y
+    CONFIG_CMD_BTRFS=y
+    CONFIG_CMD_CAT=y
+    CONFIG_DEFAULT_FDT_FILE="mediatek/mt7986a-bananapi-bpi-r3.dtb"
+    CONFIG_DISTRO_DEFAULTS=y
+    CONFIG_ENV_IS_NOWHERE=y
+    CONFIG_FS_BTRFS=y
+    CONFIG_SYS_BOOTM_LEN=0x6000000
+    CONFIG_USE_BOOTCOMMAND=y
+    CONFIG_ZSTD=y
+  '';
+  ubootVersion = "2023.07-rc3";
+  ubootSrc = pkgs.fetchurl {
+    url = "ftp://ftp.denx.de/pub/u-boot/u-boot-${ubootVersion}.tar.bz2";
+    hash = "sha256-QuwINnS9MPpMFueMP19FPAjZ9zdZWne13aWVrDoJ2C8=";
+  };
 in

-{
+rec {
  osu-lazer-bin = nix-gaming.osu-lazer-bin;
-  clang-tools_latest = pkgs.clang-tools_15;
-  clang_latest = pkgs.clang_15;
+  clang-tools_latest = pkgs.clang-tools_16;
+  clang_latest = pkgs.clang_16;
  home-daemon = callPackage ./home-daemon { };
  /*ghidra = pkgs.ghidra.overrideAttrs (old: {
    patches = old.patches ++ [ ./ghidra-stdcall.patch ];
@ -43,6 +104,81 @@ in
  # system76-scheduler = callPackage ./system76-scheduler.nix { };
  techmino = callPackage ./techmino { };

+  ubootBpiR3Sd = pkgs.buildUBoot {
+    defconfig = "mt7986a_bpir3_sd_defconfig";
+    extraConfig = ubootConfig "sd";
+    src = ubootSrc;
+    version = ubootVersion;
+    extraMeta.platforms = [ "aarch64-linux" ];
+    # https://github.com/nakato/nixos-bpir3-example/blob/main/pkgs/uboot/mt7986-default-bootcmd.patch
+    patches = [ ./mt7986-default-bootcmd.patch ];
+    filesToInstall = [ "u-boot.bin" ];
+  };
+  ubootBpiR3Emmc = pkgs.buildUBoot {
+    defconfig = "mt7986a_bpir3_emmc_defconfig";
+    extraConfig = ubootConfig "emmc";
+    src = ubootSrc;
+    version = ubootVersion;
+    extraMeta.platforms = [ "aarch64-linux" ];
+    patches = [ ./mt7986-default-bootcmd.patch ];
+    filesToInstall = [ "u-boot.bin" ];
+  };
+  armTrustedFirmwareBpiR3Sd = armTrustedFirmwareBpiR3 { uboot = ubootBpiR3Sd; bootDevice = "sdmmc"; };
+  armTrustedFirmwareBpiR3Emmc = armTrustedFirmwareBpiR3 { uboot = ubootBpiR3Emmc; bootDevice = "emmc"; };
+  bpiR3StuffCombined = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out/sd
+      mkdir -p $out/emmc
+      cp ${bpiR3StuffEmmc}/* $out/emmc
+      cp ${bpiR3StuffSd}/* $out/sd
+    '';
+    fixupPhase = "true";
+  };
+  bpiR3StuffEmmc = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff-emmc";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out
+      cp ${ubootBpiR3Emmc}/*.* $out
+      cp ${armTrustedFirmwareBpiR3Emmc}/*.* $out
+    '';
+    fixupPhase = "true";
+  };
+  bpiR3StuffSd = pkgs.stdenvNoCC.mkDerivation {
+    name = "bpi-r3-stuff-sd";
+    unpackPhase = "true";
+    buildPhase = "true";
+    installPhase = ''
+      mkdir -p $out
+      cp ${ubootBpiR3Sd}/*.* $out
+      cp ${armTrustedFirmwareBpiR3Sd}/*.* $out
+    '';
+    fixupPhase = "true";
+  };
+  linux_bpiR3 = (pkgs.buildLinux ({
+    version = "6.3";
+    modDirVersion = "6.3.0";
+
+    src = pkgs.fetchFromGitHub {
+      owner = "frank-w";
+      repo = "BPI-Router-Linux";
+      rev = "6.3-main";
+      hash = lib.fakeHash;
+    };
+
+    defconfig = "mt7986a_bpi-r3";
+  })).overrideAttrs (old: {
+    postConfigure = ''
+      sed -i "$buildRoot/.config" -e 's%^CONFIG_LOCALVERSION=.*%CONFIG_LOCALVERSION=""%'
+      sed -i "$buildRoot/include/config/auto.conf" -e 's%^CONFIG_LOCALVERSION=.*%CONFIG_LOCALVERSION=""%'
+    '';
+  });
+  linuxPackages_bpiR3 = pkgs.linuxPackagesFor linux_bpiR3;
+
  firefox-addons = lib.recurseIntoAttrs (callPackage ./firefox-addons { inherit nur sources; });
  mpvScripts = pkgs.mpvScripts // (callPackage ./mpv-scripts { });
 }
--- a/pkgs/mt7986-default-bootcmd.patch
+++ b/pkgs/mt7986-default-bootcmd.patch
@ -0,0 +1,75 @@
+diff --git a/include/configs/mt7986.h b/include/configs/mt7986.h
+index 0c41af1fc3..4ddf3cef89 100644
+--- a/include/configs/mt7986.h
+++ b/include/configs/mt7986.h
+@@ -9,6 +9,8 @@
+ #ifndef __MT7986_H
+ #define __MT7986_H
+ 
+#include <linux/stringify.h>
+
+ /* Uboot definition */
+ #define CFG_SYS_UBOOT_BASE		CONFIG_TEXT_BASE
+ 
+@@ -18,4 +20,61 @@
+ /* DRAM */
+ #define CFG_SYS_SDRAM_BASE		0x40000000
+ 
+#define BOOT_TARGET_DEVICES(func) \
+	func(MMC, mmc, 0)
+
+#include <config_distro_bootcmd.h>
+
+/*
+ * Linux mt6978a.dtsi defines reserved memory in the first 256MB:
+ * 0x4300_0000 - 0x4302_FFFF
+ * 0x4FC0_0000 - 0x4FCF_FFFF
+ * 0x4FD0_0000 - 0x4FD3_FFFF
+ * 0x4FD4_0000 - 0x4FD7_FFFF
+ * 0x4FD8_0000 - 0x4FDA_3FFF
+ *
+ * Getting uboot to obey these by adding them into the DTS here
+ * got it respecting them.  To not clobber these memory addresses
+ * fdt_high and initrd_high are being set.
+ */
+
+/*
+ * Do not relocate FDT, because relocate fails to alloc when
+ * initrd_high is set to 0x4FC0_0000 to keep initrd out of
+ * reserved memory.  This is irregardless of the reserved
+ * addresses above still being valid as far as configuraiton
+ * is concerned at that point, so I'm at a loss.
+ */
+#define	FDT_HIGH	__stringify(0xffffffffffffffff)
+/* No reloc might work now that FDT is in-place, but this should be more compatible */
+#define	INITRD_HIGH	__stringify(0x4fc00000)
+
+/* Seems to be the address uboot would relocate to so place there */
+#define	KERNEL_ADDR_R	__stringify(0x43200000)
+/* Right before the kernel, after that reserved memory, probably best spots exist */
+#define	FDT_ADDR_R	__stringify(0x43030000)
+
+#define	RAMDISK_ADDR_R	__stringify(0x4b100000)
+
+/* Anything different means no-boot */
+#define	BOOTM_SIZE	__stringify(0x10000000)
+
+/* These can be outside Linux's 256MB */
+#define	SCRIPTADDR	__stringify(0x90000000)
+#define	PXEFILE_ADDR_R	__stringify(0x90100000)
+
+
+/* No comp_addr supplied */
+#define CFG_EXTRA_ENV_SETTINGS \
+	"bootm_size=" BOOTM_SIZE "\0" \
+	"fdt_addr_r=" FDT_ADDR_R "\0" \
+	"fdt_high=" FDT_HIGH "\0" \
+	"initrd_high=" INITRD_HIGH "\0" \
+	"kernel_addr_r=" KERNEL_ADDR_R "\0" \
+	"pxefile_addr_r=" PXEFILE_ADDR_R "\0" \
+	"ramdisk_addr_r=" RAMDISK_ADDR_R "\0" \
+	"scriptaddr=" SCRIPTADDR "\0" \
+	"fdtfile=" CONFIG_DEFAULT_FDT_FILE "\0" \
+	BOOTENV
+
+ #endif
--- a/pkgs/nvfetcher.toml
+++ b/pkgs/nvfetcher.toml
@ -10,3 +10,13 @@ fetch.url = "https://github.com/FooSoft/yomichan/releases/download/$ver/yomichan
 # allow prereleases
 src.github_tag = "FastForwardTeam/FastForward"
 fetch.url = "https://github.com/FastForwardTeam/FastForward/releases/download/$ver/fastforwardteam-$ver.xpi"
+
+# nix-prefetch doesnt work with git right now for some reason, whatever
+# [atf-bpir3]
+# src.git = "https://github.com/frank-w/u-boot.git"
+# src.branch = "r3-atf"
+# src.use_commit = true
+# fetch.git = "https://github.com/frank-w/u-boot.git"
+# fetch.branch = "$ver"
+#fetch.github = "frank-w/u-boot"
+# fetch.branch = "r3-atf"
--- a/system/hardware/bpi_r3.nix
+++ b/system/hardware/bpi_r3.nix
@ -1,35 +1,26 @@
 { pkgs
-, lib
 , ... }:

-# WIP
 {
  boot.loader = {
    grub.enable = false;
    generic-extlinux-compatible.enable = true;
  };

-  # https://github.com/frank-w/BPI-Router-Linux
-  boot.kernelPackages = pkgs.linuxPackagesFor ((pkgs.buildLinux ({
-    version = "6.3";
-    modDirVersion = "6.3.0";
-
-    src = pkgs.fetchFromGitHub {
-      owner = "frank-w";
-      repo = "BPI-Router-Linux";
-      rev = "6.3-main";
-      hash = lib.fakeHash;
-    };
-
-    defconfig = "mt7986a_bpi-r3";
-  })).overrideAttrs (old: {
-    postConfigure = ''
-      sed -i "$buildRoot/.config" -e 's/^CONFIG_LOCALVERSION=.*/CONFIG_LOCALVERSION=""/'
-      sed -i "$buildRoot/include/config/auto.conf" -e 's/^CONFIG_LOCALVERSION=.*/CONFIG_LOCALVERSION=""/'
-    '';
-  }));
+  # i'm not about to build a kernel on every update without an arm device...
+  # i guess i could use my phone for building it, but no, not interested
+  boot.kernelPackages = pkgs.linuxPackages_testing;
+  # boot.kernelPackages = pkgs.linuxPackages_bpiR3;

  hardware.deviceTree.enable = true;
-  hardware.deviceTree.filter = "mt7986a-bananapi-bpi-r3*.dtb";
+  hardware.deviceTree.filter = "*mt7986*";
  hardware.enableRedistributableFirmware = true;
+
+  # # disable a bunch of useless drivers
+  # boot.initrd.includeDefaultModules = false;
+  # boot.initrd.availableKernelModules = [ "mmc_block" "dm_mod" "rfkill" "cfg80211" "mt7915e" "ubi" "sdhci" "sd_mod" "sr_mod" "xhci_pci" ];
+  boot.kernelParams = [ "console=ttyS0,115200" ];
+
+  boot.initrd.compressor = "zstd";
+  nixpkgs.buildPlatform = "x86_64-linux";
 }
--- a/system/hosts/nixmsi.nix
+++ b/system/hosts/nixmsi.nix
@ -231,5 +231,7 @@ in {
  };

  programs.ccache.enable = true;
+  services.sshd.enable = true;
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 }

--- a/system/hosts/router/default.nix
+++ b/system/hosts/router/default.nix
@ -1,9 +1,17 @@
 { config
+, pkgs
 , ... }:

+# EMMC size: 7818182656
+
 let
-  rootUuid = "00000000-0000-0000-0000-000000000000";
+  rootUuid = "44444444-4444-4444-8888-888888888888";
  rootPart = "/dev/disk/by-uuid/${rootUuid}";
+  rootfsImage = pkgs.callPackage "${pkgs.path}/nixos/lib/make-ext4-fs.nix" {
+    storePaths = config.system.build.toplevel;
+    compressImage = false;
+    volumeLabel = "NIX_ROOTFS";
+  };
 in {
  system.stateVersion = "22.11";
  # TODO
@ -13,16 +21,41 @@ in {
                options = [ "defaults" "size=2G" "mode=755" ]; };
    "/persist" =
              { device = rootPart; fsType = "btrfs"; neededForBoot = true;
-                options = [ ]; };
+                options = [ "compress=zstd:15" "subvol=@" ]; };
+    "/boot" =
+              { device = rootPart; fsType = "btrfs"; neededForBoot = true;
+                options = [ "subvol=@boot" ]; };
+    "/nix" =
+              { device = rootPart; fsType = "btrfs"; neededForBoot = true;
+                options = [ "compress=zstd:15" "subvol=@nix" ]; };
  };
+  services.openssh.enable = true;
  impermanence = {
    enable = true;
    path = /persist;
    directories = [
      { directory = /home/${config.common.mainUsername}; user = config.common.mainUsername; group = config.common.mainUsername; mode = "0700"; }
      { directory = /root; mode = "0700"; }
-      /nix
-      /boot
    ];
  };
+  hardware.wirelessRegulatoryDatabase = true;
+  system.build.emmcImage = pkgs.callPackage ./image.nix {
+    inherit config rootfsImage;
+    bpiR3Stuff = pkgs.bpiR3StuffEmmc;
+  };
+  system.build.sdImage = pkgs.callPackage ./image.nix {
+    inherit config rootfsImage;
+    bpiR3Stuff = pkgs.bpiR3StuffSd;
+  };
+  system.build.rootfs = rootfsImage;
+  boot.postBootCommands = ''
+    if [ -f ${toString config.impermanence.path}/nix-path-registration ]; then
+      ${config.nix.package.out}/bin/nix-store --load-db < ${toString config.impermanence.path}/nix-path-registration
+      mkdir -p /etc
+      touch /etc/NIXOS
+      ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+      rm -f ${toString config.impermanence.path}/nix-path-registration
+    fi
+  '';
+  boot.kernelParams = [ "boot.shell_on_fail" ];
 }
--- a/system/hosts/router/image.nix
+++ b/system/hosts/router/image.nix
@ -0,0 +1,99 @@
+{ gptfdisk
+, e2fsprogs
+, util-linux
+, zstd
+, btrfs-progs
+, vmTools
+, runCommand
+, jq
+, kmod
+, udev
+, nix
+, lkl
+
+, bpiR3Stuff
+, config
+, rootfsImage
+, ... }:
+
+let
+  imageSize = 7818182656;
+  bl2Start = 34;
+  bl2End = 8191;
+  envStart = 8192;
+  envEnd = 9215;
+  factoryStart = 9216;
+  factoryEnd = 13311;
+  fipStart = 13312;
+  fipEnd = 17407;
+  rootPartStart = fipEnd + 1;
+  rootPartEnd = builtins.floor (imageSize / 512) - 100;
+in
+
+# the vm is suuuuuper slow
+# so, do as much as possible outside of it
+# but i still need it to create subvolumes
+
+vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
+  enableParallelBuildingByDefault = true;
+  nativeBuildInputs = [
+    btrfs-progs
+    # dosfstools
+    e2fsprogs
+    util-linux # sfdisk
+    config.system.build.nixos-install
+    jq
+    kmod
+    udev
+    gptfdisk
+    zstd
+    nix
+    lkl
+  ];
+  preVM = ''
+    img=./result.img
+    ls /dev/fuse /etc/mtab || (echo "Must have /dev/fuse and /etc/mtab in nix options (extra-sandbox-paths)" && exit 1)
+    ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
+    truncate -s ${toString imageSize} $img
+
+    ${gptfdisk}/bin/sgdisk -o \
+      --set-alignment=2 \
+      -n 1:${toString bl2Start}:${toString bl2End} -c 1:bl2 -A 1:set:2 \
+      -n 2:${toString envStart}:${toString envEnd} -c 2:u-boot-env \
+      -n 3:${toString factoryStart}:${toString factoryEnd} -c 3:factory \
+      -n 4:${toString fipStart}:${toString fipEnd} -c 4:fip \
+      -n 5:${toString rootPartStart}:${toString rootPartEnd} -c 5:root -A 5:set:2 \
+      $img
+    dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
+    dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
+    truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
+    ${btrfs-progs}/bin/mkfs.btrfs \
+      --label NIXOS_SD \
+      --uuid "44444444-4444-4444-8888-888888888889" \
+      ./tmp.img
+  '';
+  # i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
+  # (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
+  postVM = ''
+    # Truncate the file at the end of the last partition
+    img=./result.img
+    mkdir -p $out/boot
+    ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
+    cp ${rootfsImage} $out/rootfs.ext4
+    cp tmp.img $out/template.btrfs
+    cp $img $out/image.img
+    echo ${toString rootPartStart} > $out/metadata
+    echo ${toString rootPartEnd} >> $out/metadata
+  '';
+  memSize = "4G";
+  QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
+} ''
+  modprobe btrfs
+  mkdir -p /mnt
+  mount -t btrfs -o space_cache=v2 /dev/vda /mnt
+  btrfs filesystem resize max /mnt
+  btrfs subvolume create /mnt/@boot
+  btrfs subvolume create /mnt/@
+  btrfs subvolume create /mnt/@nix
+  chattr +C /mnt/@boot
+'')
--- a/system/hosts/router/image.sh
+++ b/system/hosts/router/image.sh
@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# this step requires either root access or access to fuse
+# nix builder doesnt have access to either
+# i could run this in a vm, but that's slow as fuck because i'm building this on x86_64-linux
+# usage: image.sh [path to image.nix output]
+
+set -euxo pipefail
+
+(which zstd && which rsync) || exit 1
+
+userspace="$(which lklfuse >/dev/null && echo -n 1 || echo -n)"
+use_rsync="$(which rsync >/dev/null && echo -n 1 || echo -n)"
+function Mount() {
+  if [[ "$userspace" == "1" ]]; then
+    if [[ "${4-}" == "ro" ]]; then
+      lklfuse -o "type=$1" -o "$4" "$2" "$3"
+    elif [ -n "${4-}" ]; then
+      lklfuse -o "type=$1" -o "opts=$4" "$2" "$3"
+    else
+      lklfuse -o "type=$1" "$2" "$3"
+    fi
+  elif [ -n "${4-}" ]; then
+    sudo -A mount -t "$1" -o "loop,$4" "$2" "$3"
+  else
+    sudo -A mount -t "$1" -o loop "$2" "$3"
+  fi
+}
+function run() {
+  if [[ "$userspace" == "1" ]]; then
+    "$@"
+  else
+    sudo -A "$@"
+  fi
+}
+function cpr() {
+  if [[ "$use_rsync" == "1" ]]; then
+    run rsync -a --info=progress2 "$1/" "$2/"
+  else
+    run cp -rv "$1"/* "$2/"
+  fi
+}
+
+if [ -z "$1" ]; then
+  echo "missing argument"
+  exit 1
+fi
+rootfs="$1/rootfs.ext4"
+template="$1/template.btrfs"
+image="$1/image.img"
+boot="$1/boot"
+
+metadata0="$(head -n1 "$1/metadata")"
+metadata1="$(head -n2 "$1/metadata" | tail -n1)"
+if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1" ]]; then
+  echo "invalid metadata"
+  exit 1
+fi
+
+tmp=$(mktemp -d)
+cp "$template" "$tmp/template.btrfs"
+cp "$image" "$tmp/image.img"
+template="$tmp/template.btrfs"
+image="$tmp/image.img"
+chmod +w "$template" "$image"
+
+function cleanup {
+  run umount "$rootfs" || echo -n
+  run umount "$tmp/out" || echo -n
+  rm -rf "$tmp"
+}
+trap cleanup EXIT
+mkdir -p "$tmp/rootfs" "$tmp/out"
+Mount ext4 "$rootfs" "$tmp/rootfs" ro
+rootfs="$tmp/rootfs"
+Mount btrfs "$template" "$tmp/out"
+run cp -rv "$boot"/* "$tmp/out/@boot/"
+run umount "$tmp/out"
+Mount btrfs "$template" "$tmp/out" "compress=zstd:15"
+run cp -v "$rootfs/nix-path-registration" "$tmp/out/@/"
+run ls "$boot"
+cpr "$boot" "$tmp/out/@boot"
+cpr "$rootfs/nix" "$tmp/out/@nix"
+
+run umount "$rootfs"
+run umount "$tmp/out"
+
+dd conv=notrunc if="$template" of="$image" seek="$metadata0"
+
+zstd --compress "$image"
+cp "$image.zst" ./image.img.zst
--- a/system/modules/impermanence.nix
+++ b/system/modules/impermanence.nix
@ -13,6 +13,7 @@ in {
        };
        path = mkOption {
          type = types.path;
+          default = if cfg.enable then throw "You must set path to persistent storage" else "";
          description = "Default path for persistence";
        };
        directories = mkOption {