router: final fixes, it's now in production

system/hardware/bpi_r3/image.nix

@@ -2,14 +2,11 @@
 , e2fsprogs
 , util-linux
 , zstd
+, stdenvNoCC
 , btrfs-progs
 , vmTools
 , runCommand
-, jq
 , kmod
-, udev
-, nix
-, lkl
 
 , bpiR3Stuff
 , config
@@ -17,7 +14,7 @@
 , ... }:
 
 let
-  imageSize = 7818182656;
+  imageSize = 7818182656; # emmc size
   bl2Start = 34;
   bl2End = 8191;
   envStart = 8192;
@@ -33,26 +30,44 @@ in
 # the vm is suuuuuper slow
 # so, do as much as possible outside of it
 # but i still need it to create subvolumes
-
+let
-vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
+  template = vmTools.runInLinuxVM (runCommand "bpi-r3-fs-template" {
-  enableParallelBuildingByDefault = true;
-  nativeBuildInputs = [
-    btrfs-progs
-    # dosfstools
-    e2fsprogs
-    util-linux # sfdisk
-    config.system.build.nixos-install
-    jq
-    kmod
-    udev
-    gptfdisk
-    zstd
-    nix
-    lkl
-  ];
     preVM = ''
+      truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
+      ${btrfs-progs}/bin/mkfs.btrfs \
+        --label NIXOS_SD \
+        --uuid "44444444-4444-4444-8888-888888888888" \
+        ./tmp.img
+    '';
+    nativeBuildInputs = [ btrfs-progs e2fsprogs util-linux kmod ];
+    postVM = ''
+      mkdir -p $out
+      ${zstd}/bin/zstd tmp.img -o $out/template.btrfs.zst
+    '';
+    memSize = "4G";
+    QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
+  } ''
+    modprobe btrfs
+    mkdir -p /mnt
+    mount -t btrfs -o space_cache=v2 /dev/vda /mnt
+    btrfs filesystem resize max /mnt
+    btrfs subvolume create /mnt/@boot
+    btrfs subvolume create /mnt/@
+    btrfs subvolume create /mnt/@nix
+    chattr +C /mnt/@boot
+  '');
+in
+
+stdenvNoCC.mkDerivation {
+  name = "bpi-r3-fs";
+  nativeBuildInputs = [
+    util-linux # sfdisk
+    zstd
+  ];
+    # ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
+  unpackPhase = "true";
+  buildPhase = ''
     img=./result.img
-    ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
     truncate -s ${toString imageSize} $img
 
     ${gptfdisk}/bin/sgdisk -o \
@@ -65,34 +80,17 @@ vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
       $img
     dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
     dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
-    truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
-    ${btrfs-progs}/bin/mkfs.btrfs \
-      --label NIXOS_SD \
-      --uuid "44444444-4444-4444-8888-888888888888" \
-      ./tmp.img
   '';
   # i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
   # (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
-  postVM = ''
+  installPhase = ''
-    # Truncate the file at the end of the last partition
-    img=./result.img
     mkdir -p $out/boot
     ${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
-    cp ${rootfsImage} $out/rootfs.ext4
+    ln -s ${rootfsImage} $out/rootfs.ext4
-    cp tmp.img $out/template.btrfs
+    cp ${template}/template.btrfs.zst $out
-    cp $img $out/image.img
+    zstd $img -o $out/image.img.zst
     echo ${toString rootPartStart} > $out/metadata
     echo ${toString rootPartEnd} >> $out/metadata
+    ln -s ${bpiR3Stuff}/bl2.img $out/boot0.img
   '';
-  memSize = "4G";
+}
-  QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
-} ''
-  modprobe btrfs
-  mkdir -p /mnt
-  mount -t btrfs -o space_cache=v2 /dev/vda /mnt
-  btrfs filesystem resize max /mnt
-  btrfs subvolume create /mnt/@boot
-  btrfs subvolume create /mnt/@
-  btrfs subvolume create /mnt/@nix
-  chattr +C /mnt/@boot
-'')

system/hardware/bpi_r3/image.sh

@@ -46,8 +46,8 @@ if [ -z "$1" ]; then
   exit 1
 fi
 rootfs="$1/rootfs.ext4"
-template="$1/template.btrfs"
+template="$1/template.btrfs.zst"
-image="$1/image.img"
+image="$1/image.img.zst"
 boot="$1/boot"
 
 metadata0="$(head -n1 "$1/metadata")"
@@ -58,10 +58,12 @@ if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1"
 fi
 
 tmp=$(mktemp -d)
-cp "$template" "$tmp/template.btrfs"
+cp "$template" "$tmp/template.btrfs.zst"
-cp "$image" "$tmp/image.img"
+cp "$image" "$tmp/image.img.zst"
 template="$tmp/template.btrfs"
 image="$tmp/image.img"
+unzstd --rm "$template.zst"
+unzstd --rm "$image.zst"
 chmod +w "$template" "$image"
 
 function cleanup {
@@ -94,5 +96,4 @@ run umount "$tmp/out"
 
 dd conv=notrunc if="$template" of="$image" seek="$metadata0"
 
-zstd --compress "$image"
+zstd -f --rm --compress "$image" -o ./image.img.zst 
-cp "$image.zst" ./image.img.zst

system/hosts/nixserver/default.nix

@@ -85,9 +85,6 @@ in {
   };
   console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz";
   networking.useDHCP = true;
-  networking.resolvconf.extraConfig = ''
-    name_servers="127.0.0.1 ::1"
-  '';
   networking.firewall = {
     enable = true;
     allowedTCPPorts = [
@@ -107,12 +104,15 @@ in {
   # UNBOUND
   users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ];
 
+  #networking.resolvconf.extraConfig = ''
+  #  name_servers="127.0.0.1 ::1"
+  #'';
   services.unbound = {
-    enable = true;
+    enable = false;
     package = pkgs.unbound-with-systemd.override {
       stdenv = pkgs.ccacheStdenv;
       withPythonModule = true;
-      python = unbound-python;
+      python = pkgs.python3;
     };
     localControlSocketPath = "/run/unbound/unbound.ctl";
     resolveLocalQueries = false;

system/hosts/router/avahi-resolver-v2.py

@@ -681,7 +681,9 @@ def operate(id, event, qstate, qdata):
                         f.write(new_data)
             if changed4:
                 for qname in qnames:
-                    name4 = NFT_QUERIES[qname]['name4']
+                    q = NFT_QUERIES[qname]
+                    name4 = q['name4']
+                    ips4 = q['ips4']
                     if name4:
                         ip2 = []
                         for ip in ip4:
@@ -698,7 +700,9 @@ def operate(id, event, qstate, qdata):
                             add_ips(name4, False, ip2)
             if changed6:
                 for qname in qnames:
-                    name6 = NFT_QUERIES[qname]['name6']
+                    q = NFT_QUERIES[qname]
+                    name6 = q['name6']
+                    ips6 = q['ips6']
                     if name6:
                         ip2 = []
                         for ip in ip6:

system/hosts/router/default.nix

@@ -98,7 +98,8 @@ let
         [return];
 
       ingress_lan_common = add chain
-        [(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
+        # there are some issues with this, disable it for lan
+        # [(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
         [(jump "ingress_common")];
 
       ingress_wan_common = add chain
@@ -151,8 +152,8 @@ let
           }
           // lib.genAttrs lans (_: jump "inbound_lan_common")
           // lib.genAttrs wans (_: jump "inbound_wan_common")
-        ))]
+        ))];
-        [(log "${logPrefix}inbound drop ")];
+        #[(log "${logPrefix}inbound drop ")];
 
       forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; }
         [(vmap ct.state { established = accept; related = accept; invalid = drop; })]
@@ -364,7 +365,7 @@ in {
       { service = "wireguard-wg0"; inNetns = false; }
     ];
     systemdLinkLinkConfig.MACAddressPolicy = "none";
-    systemdLinkLinkConfig.MACAddress = "11:22:33:44:55:66";
+    systemdLinkLinkConfig.MACAddress = cfg.routerMac;
     dhcpcd.enable = true;
     networkNamespace = "wan";
   };
@@ -408,6 +409,34 @@ in {
     ipv4.kea.enable = true;
     ipv6.radvd.enable = true;
     ipv6.kea.enable = true;
+  };
+  router.networkNamespaces.default = {
+    # set routing table depending on packet mark
+    rules = [
+      { ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
+      { ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
+      # don't add vpn_table as it should be the default
+      { ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
+      { ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
+    ] ++ builtins.concatLists (map (rule: let
+      table = if rule.inVpn then 0 else wan_table;
+      forEachPort = func: port:
+        if builtins.isInt port then [ (func port) ]
+        else if port?set then builtins.concatLists (map (forEachPort func) port.set)
+        else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
+        else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
+        else throw "Unsupported expr: ${builtins.toJSON port}";
+      gen = len: proto: tgt:
+        forEachPort
+          (port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
+          (if tgt.port == null then rule.port else tgt.port);
+    in   lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32  "tcp" rule.target4))
+      ++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32  "udp" rule.target4))
+      ++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true;  extraArgs = x; }) (gen 128 "tcp" rule.target6))
+      ++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true;  extraArgs = x; }) (gen 128 "udp" rule.target6))
+
+    ) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
+
     nftables.stopJsonRules = mkFlushRules {};
     nftables.jsonRules = mkRules {
       selfIp4 = gatewayAddr4;
@@ -541,6 +570,8 @@ in {
     ipv6.routes = [
       { extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; }
     ];
+  };
+  router.networkNamespaces.wan = {
     nftables.stopJsonRules = mkFlushRules {};
     nftables.jsonRules = mkRules {
       selfIp4 = wanNetnsAddr4;
@@ -600,30 +631,6 @@ in {
       ];
     };
   };
-  router.networkNamespaces.default.rules = [
-    { ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
-    { ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
-    # don't add vpn_table as it should be the default
-    { ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
-    { ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
-  ] ++ builtins.concatLists (map (rule: let
-    table = if rule.inVpn then 0 else wan_table;
-    forEachPort = func: port:
-      if builtins.isInt port then [ (func port) ]
-      else if port?set then builtins.concatLists (map (forEachPort func) port.set)
-      else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
-      else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
-      else throw "Unsupported expr: ${builtins.toJSON port}";
-    gen = len: proto: tgt:
-      forEachPort
-        (port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
-        (if tgt.port == null then rule.port else tgt.port);
-  in   lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32  "tcp" rule.target4))
-    ++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32  "udp" rule.target4))
-    ++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true;  extraArgs = x; }) (gen 128 "tcp" rule.target6))
-    ++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true;  extraArgs = x; }) (gen 128 "udp" rule.target6))
-
-  ) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
 
   networking.wireguard.interfaces.wg0 = cfg.wireguard // {
     socketNamespace = "wan";
@@ -673,9 +680,9 @@ in {
     # load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
     # resolve domains and append it to ips and add it to the nftables sets
     environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6";
-    # it needs to run after br0 has been set up because it sets up the sets
+    # it needs to run after nftables has been set up because it sets up the sets
-    after = [ "nftables-br0.service" ];
+    after = [ "nftables-default.service" ];
-    wants = [ "nftables-br0.service" ];
+    wants = [ "nftables-default.service" ];
     # allow it to call nft
     serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ];
   };

system/hosts/router/options.nix

@@ -5,6 +5,10 @@
 
 {
   options.router-settings = {
+    routerMac = lib.mkOption {
+      description = "router's mac address";
+      type = lib.types.str;
+    };
     serverMac = lib.mkOption {
       description = "server's mac address";
       type = lib.types.str;