router: final fixes, it's now in production

This commit is contained in:
chayleaf 2023-06-24 09:49:57 +07:00
parent 8894e0d89c
commit 90a12a5ed0
6 changed files with 100 additions and 86 deletions

View file

@ -2,14 +2,11 @@
, e2fsprogs , e2fsprogs
, util-linux , util-linux
, zstd , zstd
, stdenvNoCC
, btrfs-progs , btrfs-progs
, vmTools , vmTools
, runCommand , runCommand
, jq
, kmod , kmod
, udev
, nix
, lkl
, bpiR3Stuff , bpiR3Stuff
, config , config
@ -17,7 +14,7 @@
, ... }: , ... }:
let let
imageSize = 7818182656; imageSize = 7818182656; # emmc size
bl2Start = 34; bl2Start = 34;
bl2End = 8191; bl2End = 8191;
envStart = 8192; envStart = 8192;
@ -33,56 +30,19 @@ in
# the vm is suuuuuper slow # the vm is suuuuuper slow
# so, do as much as possible outside of it # so, do as much as possible outside of it
# but i still need it to create subvolumes # but i still need it to create subvolumes
let
vmTools.runInLinuxVM (runCommand "bpi-r3-fs" { template = vmTools.runInLinuxVM (runCommand "bpi-r3-fs-template" {
enableParallelBuildingByDefault = true;
nativeBuildInputs = [
btrfs-progs
# dosfstools
e2fsprogs
util-linux # sfdisk
config.system.build.nixos-install
jq
kmod
udev
gptfdisk
zstd
nix
lkl
];
preVM = '' preVM = ''
img=./result.img
${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
truncate -s ${toString imageSize} $img
${gptfdisk}/bin/sgdisk -o \
--set-alignment=2 \
-n 1:${toString bl2Start}:${toString bl2End} -c 1:bl2 -A 1:set:2 \
-n 2:${toString envStart}:${toString envEnd} -c 2:u-boot-env \
-n 3:${toString factoryStart}:${toString factoryEnd} -c 3:factory \
-n 4:${toString fipStart}:${toString fipEnd} -c 4:fip \
-n 5:${toString rootPartStart}:${toString rootPartEnd} -c 5:root -A 5:set:2 \
$img
dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
${btrfs-progs}/bin/mkfs.btrfs \ ${btrfs-progs}/bin/mkfs.btrfs \
--label NIXOS_SD \ --label NIXOS_SD \
--uuid "44444444-4444-4444-8888-888888888888" \ --uuid "44444444-4444-4444-8888-888888888888" \
./tmp.img ./tmp.img
''; '';
# i give up on making this work in a nix derivation, just do the rest in a script (./image.sh) nativeBuildInputs = [ btrfs-progs e2fsprogs util-linux kmod ];
# (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
postVM = '' postVM = ''
# Truncate the file at the end of the last partition mkdir -p $out
img=./result.img ${zstd}/bin/zstd tmp.img -o $out/template.btrfs.zst
mkdir -p $out/boot
${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
cp ${rootfsImage} $out/rootfs.ext4
cp tmp.img $out/template.btrfs
cp $img $out/image.img
echo ${toString rootPartStart} > $out/metadata
echo ${toString rootPartEnd} >> $out/metadata
''; '';
memSize = "4G"; memSize = "4G";
QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm"; QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
@ -95,4 +55,42 @@ vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
btrfs subvolume create /mnt/@ btrfs subvolume create /mnt/@
btrfs subvolume create /mnt/@nix btrfs subvolume create /mnt/@nix
chattr +C /mnt/@boot chattr +C /mnt/@boot
'') '');
in
stdenvNoCC.mkDerivation {
name = "bpi-r3-fs";
nativeBuildInputs = [
util-linux # sfdisk
zstd
];
# ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
unpackPhase = "true";
buildPhase = ''
img=./result.img
truncate -s ${toString imageSize} $img
${gptfdisk}/bin/sgdisk -o \
--set-alignment=2 \
-n 1:${toString bl2Start}:${toString bl2End} -c 1:bl2 -A 1:set:2 \
-n 2:${toString envStart}:${toString envEnd} -c 2:u-boot-env \
-n 3:${toString factoryStart}:${toString factoryEnd} -c 3:factory \
-n 4:${toString fipStart}:${toString fipEnd} -c 4:fip \
-n 5:${toString rootPartStart}:${toString rootPartEnd} -c 5:root -A 5:set:2 \
$img
dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
'';
# i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
# (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
installPhase = ''
mkdir -p $out/boot
${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
ln -s ${rootfsImage} $out/rootfs.ext4
cp ${template}/template.btrfs.zst $out
zstd $img -o $out/image.img.zst
echo ${toString rootPartStart} > $out/metadata
echo ${toString rootPartEnd} >> $out/metadata
ln -s ${bpiR3Stuff}/bl2.img $out/boot0.img
'';
}

View file

@ -46,8 +46,8 @@ if [ -z "$1" ]; then
exit 1 exit 1
fi fi
rootfs="$1/rootfs.ext4" rootfs="$1/rootfs.ext4"
template="$1/template.btrfs" template="$1/template.btrfs.zst"
image="$1/image.img" image="$1/image.img.zst"
boot="$1/boot" boot="$1/boot"
metadata0="$(head -n1 "$1/metadata")" metadata0="$(head -n1 "$1/metadata")"
@ -58,10 +58,12 @@ if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1"
fi fi
tmp=$(mktemp -d) tmp=$(mktemp -d)
cp "$template" "$tmp/template.btrfs" cp "$template" "$tmp/template.btrfs.zst"
cp "$image" "$tmp/image.img" cp "$image" "$tmp/image.img.zst"
template="$tmp/template.btrfs" template="$tmp/template.btrfs"
image="$tmp/image.img" image="$tmp/image.img"
unzstd --rm "$template.zst"
unzstd --rm "$image.zst"
chmod +w "$template" "$image" chmod +w "$template" "$image"
function cleanup { function cleanup {
@ -94,5 +96,4 @@ run umount "$tmp/out"
dd conv=notrunc if="$template" of="$image" seek="$metadata0" dd conv=notrunc if="$template" of="$image" seek="$metadata0"
zstd --compress "$image" zstd -f --rm --compress "$image" -o ./image.img.zst
cp "$image.zst" ./image.img.zst

View file

@ -85,9 +85,6 @@ in {
}; };
console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz"; console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz";
networking.useDHCP = true; networking.useDHCP = true;
networking.resolvconf.extraConfig = ''
name_servers="127.0.0.1 ::1"
'';
networking.firewall = { networking.firewall = {
enable = true; enable = true;
allowedTCPPorts = [ allowedTCPPorts = [
@ -107,12 +104,15 @@ in {
# UNBOUND # UNBOUND
users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ]; users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ];
#networking.resolvconf.extraConfig = ''
# name_servers="127.0.0.1 ::1"
#'';
services.unbound = { services.unbound = {
enable = true; enable = false;
package = pkgs.unbound-with-systemd.override { package = pkgs.unbound-with-systemd.override {
stdenv = pkgs.ccacheStdenv; stdenv = pkgs.ccacheStdenv;
withPythonModule = true; withPythonModule = true;
python = unbound-python; python = pkgs.python3;
}; };
localControlSocketPath = "/run/unbound/unbound.ctl"; localControlSocketPath = "/run/unbound/unbound.ctl";
resolveLocalQueries = false; resolveLocalQueries = false;

View file

@ -681,7 +681,9 @@ def operate(id, event, qstate, qdata):
f.write(new_data) f.write(new_data)
if changed4: if changed4:
for qname in qnames: for qname in qnames:
name4 = NFT_QUERIES[qname]['name4'] q = NFT_QUERIES[qname]
name4 = q['name4']
ips4 = q['ips4']
if name4: if name4:
ip2 = [] ip2 = []
for ip in ip4: for ip in ip4:
@ -698,7 +700,9 @@ def operate(id, event, qstate, qdata):
add_ips(name4, False, ip2) add_ips(name4, False, ip2)
if changed6: if changed6:
for qname in qnames: for qname in qnames:
name6 = NFT_QUERIES[qname]['name6'] q = NFT_QUERIES[qname]
name6 = q['name6']
ips6 = q['ips6']
if name6: if name6:
ip2 = [] ip2 = []
for ip in ip6: for ip in ip6:

View file

@ -98,7 +98,8 @@ let
[return]; [return];
ingress_lan_common = add chain ingress_lan_common = add chain
[(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop] # there are some issues with this, disable it for lan
# [(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
[(jump "ingress_common")]; [(jump "ingress_common")];
ingress_wan_common = add chain ingress_wan_common = add chain
@ -151,8 +152,8 @@ let
} }
// lib.genAttrs lans (_: jump "inbound_lan_common") // lib.genAttrs lans (_: jump "inbound_lan_common")
// lib.genAttrs wans (_: jump "inbound_wan_common") // lib.genAttrs wans (_: jump "inbound_wan_common")
))] ))];
[(log "${logPrefix}inbound drop ")]; #[(log "${logPrefix}inbound drop ")];
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; } forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; }
[(vmap ct.state { established = accept; related = accept; invalid = drop; })] [(vmap ct.state { established = accept; related = accept; invalid = drop; })]
@ -364,7 +365,7 @@ in {
{ service = "wireguard-wg0"; inNetns = false; } { service = "wireguard-wg0"; inNetns = false; }
]; ];
systemdLinkLinkConfig.MACAddressPolicy = "none"; systemdLinkLinkConfig.MACAddressPolicy = "none";
systemdLinkLinkConfig.MACAddress = "11:22:33:44:55:66"; systemdLinkLinkConfig.MACAddress = cfg.routerMac;
dhcpcd.enable = true; dhcpcd.enable = true;
networkNamespace = "wan"; networkNamespace = "wan";
}; };
@ -408,6 +409,34 @@ in {
ipv4.kea.enable = true; ipv4.kea.enable = true;
ipv6.radvd.enable = true; ipv6.radvd.enable = true;
ipv6.kea.enable = true; ipv6.kea.enable = true;
};
router.networkNamespaces.default = {
# set routing table depending on packet mark
rules = [
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
# don't add vpn_table as it should be the default
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
] ++ builtins.concatLists (map (rule: let
table = if rule.inVpn then 0 else wan_table;
forEachPort = func: port:
if builtins.isInt port then [ (func port) ]
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
else throw "Unsupported expr: ${builtins.toJSON port}";
gen = len: proto: tgt:
forEachPort
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
(if tgt.port == null then rule.port else tgt.port);
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
nftables.stopJsonRules = mkFlushRules {}; nftables.stopJsonRules = mkFlushRules {};
nftables.jsonRules = mkRules { nftables.jsonRules = mkRules {
selfIp4 = gatewayAddr4; selfIp4 = gatewayAddr4;
@ -541,6 +570,8 @@ in {
ipv6.routes = [ ipv6.routes = [
{ extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; } { extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; }
]; ];
};
router.networkNamespaces.wan = {
nftables.stopJsonRules = mkFlushRules {}; nftables.stopJsonRules = mkFlushRules {};
nftables.jsonRules = mkRules { nftables.jsonRules = mkRules {
selfIp4 = wanNetnsAddr4; selfIp4 = wanNetnsAddr4;
@ -600,30 +631,6 @@ in {
]; ];
}; };
}; };
router.networkNamespaces.default.rules = [
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
# don't add vpn_table as it should be the default
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
] ++ builtins.concatLists (map (rule: let
table = if rule.inVpn then 0 else wan_table;
forEachPort = func: port:
if builtins.isInt port then [ (func port) ]
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
else throw "Unsupported expr: ${builtins.toJSON port}";
gen = len: proto: tgt:
forEachPort
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
(if tgt.port == null then rule.port else tgt.port);
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
networking.wireguard.interfaces.wg0 = cfg.wireguard // { networking.wireguard.interfaces.wg0 = cfg.wireguard // {
socketNamespace = "wan"; socketNamespace = "wan";
@ -673,9 +680,9 @@ in {
# load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json # load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
# resolve domains and append it to ips and add it to the nftables sets # resolve domains and append it to ips and add it to the nftables sets
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6"; environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6";
# it needs to run after br0 has been set up because it sets up the sets # it needs to run after nftables has been set up because it sets up the sets
after = [ "nftables-br0.service" ]; after = [ "nftables-default.service" ];
wants = [ "nftables-br0.service" ]; wants = [ "nftables-default.service" ];
# allow it to call nft # allow it to call nft
serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ]; serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ];
}; };

View file

@ -5,6 +5,10 @@
{ {
options.router-settings = { options.router-settings = {
routerMac = lib.mkOption {
description = "router's mac address";
type = lib.types.str;
};
serverMac = lib.mkOption { serverMac = lib.mkOption {
description = "server's mac address"; description = "server's mac address";
type = lib.types.str; type = lib.types.str;