router: final fixes, it's now in production
This commit is contained in:
parent
8894e0d89c
commit
90a12a5ed0
|
@ -2,14 +2,11 @@
|
||||||
, e2fsprogs
|
, e2fsprogs
|
||||||
, util-linux
|
, util-linux
|
||||||
, zstd
|
, zstd
|
||||||
|
, stdenvNoCC
|
||||||
, btrfs-progs
|
, btrfs-progs
|
||||||
, vmTools
|
, vmTools
|
||||||
, runCommand
|
, runCommand
|
||||||
, jq
|
|
||||||
, kmod
|
, kmod
|
||||||
, udev
|
|
||||||
, nix
|
|
||||||
, lkl
|
|
||||||
|
|
||||||
, bpiR3Stuff
|
, bpiR3Stuff
|
||||||
, config
|
, config
|
||||||
|
@ -17,7 +14,7 @@
|
||||||
, ... }:
|
, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
imageSize = 7818182656;
|
imageSize = 7818182656; # emmc size
|
||||||
bl2Start = 34;
|
bl2Start = 34;
|
||||||
bl2End = 8191;
|
bl2End = 8191;
|
||||||
envStart = 8192;
|
envStart = 8192;
|
||||||
|
@ -33,26 +30,44 @@ in
|
||||||
# the vm is suuuuuper slow
|
# the vm is suuuuuper slow
|
||||||
# so, do as much as possible outside of it
|
# so, do as much as possible outside of it
|
||||||
# but i still need it to create subvolumes
|
# but i still need it to create subvolumes
|
||||||
|
let
|
||||||
vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
|
template = vmTools.runInLinuxVM (runCommand "bpi-r3-fs-template" {
|
||||||
enableParallelBuildingByDefault = true;
|
|
||||||
nativeBuildInputs = [
|
|
||||||
btrfs-progs
|
|
||||||
# dosfstools
|
|
||||||
e2fsprogs
|
|
||||||
util-linux # sfdisk
|
|
||||||
config.system.build.nixos-install
|
|
||||||
jq
|
|
||||||
kmod
|
|
||||||
udev
|
|
||||||
gptfdisk
|
|
||||||
zstd
|
|
||||||
nix
|
|
||||||
lkl
|
|
||||||
];
|
|
||||||
preVM = ''
|
preVM = ''
|
||||||
|
truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
|
||||||
|
${btrfs-progs}/bin/mkfs.btrfs \
|
||||||
|
--label NIXOS_SD \
|
||||||
|
--uuid "44444444-4444-4444-8888-888888888888" \
|
||||||
|
./tmp.img
|
||||||
|
'';
|
||||||
|
nativeBuildInputs = [ btrfs-progs e2fsprogs util-linux kmod ];
|
||||||
|
postVM = ''
|
||||||
|
mkdir -p $out
|
||||||
|
${zstd}/bin/zstd tmp.img -o $out/template.btrfs.zst
|
||||||
|
'';
|
||||||
|
memSize = "4G";
|
||||||
|
QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
|
||||||
|
} ''
|
||||||
|
modprobe btrfs
|
||||||
|
mkdir -p /mnt
|
||||||
|
mount -t btrfs -o space_cache=v2 /dev/vda /mnt
|
||||||
|
btrfs filesystem resize max /mnt
|
||||||
|
btrfs subvolume create /mnt/@boot
|
||||||
|
btrfs subvolume create /mnt/@
|
||||||
|
btrfs subvolume create /mnt/@nix
|
||||||
|
chattr +C /mnt/@boot
|
||||||
|
'');
|
||||||
|
in
|
||||||
|
|
||||||
|
stdenvNoCC.mkDerivation {
|
||||||
|
name = "bpi-r3-fs";
|
||||||
|
nativeBuildInputs = [
|
||||||
|
util-linux # sfdisk
|
||||||
|
zstd
|
||||||
|
];
|
||||||
|
# ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
|
||||||
|
unpackPhase = "true";
|
||||||
|
buildPhase = ''
|
||||||
img=./result.img
|
img=./result.img
|
||||||
${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
|
|
||||||
truncate -s ${toString imageSize} $img
|
truncate -s ${toString imageSize} $img
|
||||||
|
|
||||||
${gptfdisk}/bin/sgdisk -o \
|
${gptfdisk}/bin/sgdisk -o \
|
||||||
|
@ -65,34 +80,17 @@ vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
|
||||||
$img
|
$img
|
||||||
dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
|
dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
|
||||||
dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
|
dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
|
||||||
truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
|
|
||||||
${btrfs-progs}/bin/mkfs.btrfs \
|
|
||||||
--label NIXOS_SD \
|
|
||||||
--uuid "44444444-4444-4444-8888-888888888888" \
|
|
||||||
./tmp.img
|
|
||||||
'';
|
'';
|
||||||
# i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
|
# i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
|
||||||
# (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
|
# (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
|
||||||
postVM = ''
|
installPhase = ''
|
||||||
# Truncate the file at the end of the last partition
|
|
||||||
img=./result.img
|
|
||||||
mkdir -p $out/boot
|
mkdir -p $out/boot
|
||||||
${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
|
${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
|
||||||
cp ${rootfsImage} $out/rootfs.ext4
|
ln -s ${rootfsImage} $out/rootfs.ext4
|
||||||
cp tmp.img $out/template.btrfs
|
cp ${template}/template.btrfs.zst $out
|
||||||
cp $img $out/image.img
|
zstd $img -o $out/image.img.zst
|
||||||
echo ${toString rootPartStart} > $out/metadata
|
echo ${toString rootPartStart} > $out/metadata
|
||||||
echo ${toString rootPartEnd} >> $out/metadata
|
echo ${toString rootPartEnd} >> $out/metadata
|
||||||
|
ln -s ${bpiR3Stuff}/bl2.img $out/boot0.img
|
||||||
'';
|
'';
|
||||||
memSize = "4G";
|
}
|
||||||
QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
|
|
||||||
} ''
|
|
||||||
modprobe btrfs
|
|
||||||
mkdir -p /mnt
|
|
||||||
mount -t btrfs -o space_cache=v2 /dev/vda /mnt
|
|
||||||
btrfs filesystem resize max /mnt
|
|
||||||
btrfs subvolume create /mnt/@boot
|
|
||||||
btrfs subvolume create /mnt/@
|
|
||||||
btrfs subvolume create /mnt/@nix
|
|
||||||
chattr +C /mnt/@boot
|
|
||||||
'')
|
|
||||||
|
|
|
@ -46,8 +46,8 @@ if [ -z "$1" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
rootfs="$1/rootfs.ext4"
|
rootfs="$1/rootfs.ext4"
|
||||||
template="$1/template.btrfs"
|
template="$1/template.btrfs.zst"
|
||||||
image="$1/image.img"
|
image="$1/image.img.zst"
|
||||||
boot="$1/boot"
|
boot="$1/boot"
|
||||||
|
|
||||||
metadata0="$(head -n1 "$1/metadata")"
|
metadata0="$(head -n1 "$1/metadata")"
|
||||||
|
@ -58,10 +58,12 @@ if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
tmp=$(mktemp -d)
|
tmp=$(mktemp -d)
|
||||||
cp "$template" "$tmp/template.btrfs"
|
cp "$template" "$tmp/template.btrfs.zst"
|
||||||
cp "$image" "$tmp/image.img"
|
cp "$image" "$tmp/image.img.zst"
|
||||||
template="$tmp/template.btrfs"
|
template="$tmp/template.btrfs"
|
||||||
image="$tmp/image.img"
|
image="$tmp/image.img"
|
||||||
|
unzstd --rm "$template.zst"
|
||||||
|
unzstd --rm "$image.zst"
|
||||||
chmod +w "$template" "$image"
|
chmod +w "$template" "$image"
|
||||||
|
|
||||||
function cleanup {
|
function cleanup {
|
||||||
|
@ -94,5 +96,4 @@ run umount "$tmp/out"
|
||||||
|
|
||||||
dd conv=notrunc if="$template" of="$image" seek="$metadata0"
|
dd conv=notrunc if="$template" of="$image" seek="$metadata0"
|
||||||
|
|
||||||
zstd --compress "$image"
|
zstd -f --rm --compress "$image" -o ./image.img.zst
|
||||||
cp "$image.zst" ./image.img.zst
|
|
||||||
|
|
|
@ -85,9 +85,6 @@ in {
|
||||||
};
|
};
|
||||||
console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz";
|
console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz";
|
||||||
networking.useDHCP = true;
|
networking.useDHCP = true;
|
||||||
networking.resolvconf.extraConfig = ''
|
|
||||||
name_servers="127.0.0.1 ::1"
|
|
||||||
'';
|
|
||||||
networking.firewall = {
|
networking.firewall = {
|
||||||
enable = true;
|
enable = true;
|
||||||
allowedTCPPorts = [
|
allowedTCPPorts = [
|
||||||
|
@ -107,12 +104,15 @@ in {
|
||||||
# UNBOUND
|
# UNBOUND
|
||||||
users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ];
|
users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ];
|
||||||
|
|
||||||
|
#networking.resolvconf.extraConfig = ''
|
||||||
|
# name_servers="127.0.0.1 ::1"
|
||||||
|
#'';
|
||||||
services.unbound = {
|
services.unbound = {
|
||||||
enable = true;
|
enable = false;
|
||||||
package = pkgs.unbound-with-systemd.override {
|
package = pkgs.unbound-with-systemd.override {
|
||||||
stdenv = pkgs.ccacheStdenv;
|
stdenv = pkgs.ccacheStdenv;
|
||||||
withPythonModule = true;
|
withPythonModule = true;
|
||||||
python = unbound-python;
|
python = pkgs.python3;
|
||||||
};
|
};
|
||||||
localControlSocketPath = "/run/unbound/unbound.ctl";
|
localControlSocketPath = "/run/unbound/unbound.ctl";
|
||||||
resolveLocalQueries = false;
|
resolveLocalQueries = false;
|
||||||
|
|
|
@ -681,7 +681,9 @@ def operate(id, event, qstate, qdata):
|
||||||
f.write(new_data)
|
f.write(new_data)
|
||||||
if changed4:
|
if changed4:
|
||||||
for qname in qnames:
|
for qname in qnames:
|
||||||
name4 = NFT_QUERIES[qname]['name4']
|
q = NFT_QUERIES[qname]
|
||||||
|
name4 = q['name4']
|
||||||
|
ips4 = q['ips4']
|
||||||
if name4:
|
if name4:
|
||||||
ip2 = []
|
ip2 = []
|
||||||
for ip in ip4:
|
for ip in ip4:
|
||||||
|
@ -698,7 +700,9 @@ def operate(id, event, qstate, qdata):
|
||||||
add_ips(name4, False, ip2)
|
add_ips(name4, False, ip2)
|
||||||
if changed6:
|
if changed6:
|
||||||
for qname in qnames:
|
for qname in qnames:
|
||||||
name6 = NFT_QUERIES[qname]['name6']
|
q = NFT_QUERIES[qname]
|
||||||
|
name6 = q['name6']
|
||||||
|
ips6 = q['ips6']
|
||||||
if name6:
|
if name6:
|
||||||
ip2 = []
|
ip2 = []
|
||||||
for ip in ip6:
|
for ip in ip6:
|
||||||
|
|
|
@ -98,7 +98,8 @@ let
|
||||||
[return];
|
[return];
|
||||||
|
|
||||||
ingress_lan_common = add chain
|
ingress_lan_common = add chain
|
||||||
[(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
|
# there are some issues with this, disable it for lan
|
||||||
|
# [(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
|
||||||
[(jump "ingress_common")];
|
[(jump "ingress_common")];
|
||||||
|
|
||||||
ingress_wan_common = add chain
|
ingress_wan_common = add chain
|
||||||
|
@ -151,8 +152,8 @@ let
|
||||||
}
|
}
|
||||||
// lib.genAttrs lans (_: jump "inbound_lan_common")
|
// lib.genAttrs lans (_: jump "inbound_lan_common")
|
||||||
// lib.genAttrs wans (_: jump "inbound_wan_common")
|
// lib.genAttrs wans (_: jump "inbound_wan_common")
|
||||||
))]
|
))];
|
||||||
[(log "${logPrefix}inbound drop ")];
|
#[(log "${logPrefix}inbound drop ")];
|
||||||
|
|
||||||
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; }
|
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; }
|
||||||
[(vmap ct.state { established = accept; related = accept; invalid = drop; })]
|
[(vmap ct.state { established = accept; related = accept; invalid = drop; })]
|
||||||
|
@ -364,7 +365,7 @@ in {
|
||||||
{ service = "wireguard-wg0"; inNetns = false; }
|
{ service = "wireguard-wg0"; inNetns = false; }
|
||||||
];
|
];
|
||||||
systemdLinkLinkConfig.MACAddressPolicy = "none";
|
systemdLinkLinkConfig.MACAddressPolicy = "none";
|
||||||
systemdLinkLinkConfig.MACAddress = "11:22:33:44:55:66";
|
systemdLinkLinkConfig.MACAddress = cfg.routerMac;
|
||||||
dhcpcd.enable = true;
|
dhcpcd.enable = true;
|
||||||
networkNamespace = "wan";
|
networkNamespace = "wan";
|
||||||
};
|
};
|
||||||
|
@ -408,6 +409,34 @@ in {
|
||||||
ipv4.kea.enable = true;
|
ipv4.kea.enable = true;
|
||||||
ipv6.radvd.enable = true;
|
ipv6.radvd.enable = true;
|
||||||
ipv6.kea.enable = true;
|
ipv6.kea.enable = true;
|
||||||
|
};
|
||||||
|
router.networkNamespaces.default = {
|
||||||
|
# set routing table depending on packet mark
|
||||||
|
rules = [
|
||||||
|
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
|
||||||
|
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
|
||||||
|
# don't add vpn_table as it should be the default
|
||||||
|
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
|
||||||
|
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
|
||||||
|
] ++ builtins.concatLists (map (rule: let
|
||||||
|
table = if rule.inVpn then 0 else wan_table;
|
||||||
|
forEachPort = func: port:
|
||||||
|
if builtins.isInt port then [ (func port) ]
|
||||||
|
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
|
||||||
|
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
|
||||||
|
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
|
||||||
|
else throw "Unsupported expr: ${builtins.toJSON port}";
|
||||||
|
gen = len: proto: tgt:
|
||||||
|
forEachPort
|
||||||
|
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
|
||||||
|
(if tgt.port == null then rule.port else tgt.port);
|
||||||
|
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
|
||||||
|
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
|
||||||
|
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
|
||||||
|
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
|
||||||
|
|
||||||
|
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
|
||||||
|
|
||||||
nftables.stopJsonRules = mkFlushRules {};
|
nftables.stopJsonRules = mkFlushRules {};
|
||||||
nftables.jsonRules = mkRules {
|
nftables.jsonRules = mkRules {
|
||||||
selfIp4 = gatewayAddr4;
|
selfIp4 = gatewayAddr4;
|
||||||
|
@ -541,6 +570,8 @@ in {
|
||||||
ipv6.routes = [
|
ipv6.routes = [
|
||||||
{ extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; }
|
{ extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; }
|
||||||
];
|
];
|
||||||
|
};
|
||||||
|
router.networkNamespaces.wan = {
|
||||||
nftables.stopJsonRules = mkFlushRules {};
|
nftables.stopJsonRules = mkFlushRules {};
|
||||||
nftables.jsonRules = mkRules {
|
nftables.jsonRules = mkRules {
|
||||||
selfIp4 = wanNetnsAddr4;
|
selfIp4 = wanNetnsAddr4;
|
||||||
|
@ -600,30 +631,6 @@ in {
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
router.networkNamespaces.default.rules = [
|
|
||||||
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
|
|
||||||
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
|
|
||||||
# don't add vpn_table as it should be the default
|
|
||||||
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
|
|
||||||
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
|
|
||||||
] ++ builtins.concatLists (map (rule: let
|
|
||||||
table = if rule.inVpn then 0 else wan_table;
|
|
||||||
forEachPort = func: port:
|
|
||||||
if builtins.isInt port then [ (func port) ]
|
|
||||||
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
|
|
||||||
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
|
|
||||||
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
|
|
||||||
else throw "Unsupported expr: ${builtins.toJSON port}";
|
|
||||||
gen = len: proto: tgt:
|
|
||||||
forEachPort
|
|
||||||
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
|
|
||||||
(if tgt.port == null then rule.port else tgt.port);
|
|
||||||
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
|
|
||||||
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
|
|
||||||
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
|
|
||||||
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
|
|
||||||
|
|
||||||
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
|
|
||||||
|
|
||||||
networking.wireguard.interfaces.wg0 = cfg.wireguard // {
|
networking.wireguard.interfaces.wg0 = cfg.wireguard // {
|
||||||
socketNamespace = "wan";
|
socketNamespace = "wan";
|
||||||
|
@ -673,9 +680,9 @@ in {
|
||||||
# load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
|
# load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
|
||||||
# resolve domains and append it to ips and add it to the nftables sets
|
# resolve domains and append it to ips and add it to the nftables sets
|
||||||
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6";
|
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6";
|
||||||
# it needs to run after br0 has been set up because it sets up the sets
|
# it needs to run after nftables has been set up because it sets up the sets
|
||||||
after = [ "nftables-br0.service" ];
|
after = [ "nftables-default.service" ];
|
||||||
wants = [ "nftables-br0.service" ];
|
wants = [ "nftables-default.service" ];
|
||||||
# allow it to call nft
|
# allow it to call nft
|
||||||
serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ];
|
serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ];
|
||||||
};
|
};
|
||||||
|
|
|
@ -5,6 +5,10 @@
|
||||||
|
|
||||||
{
|
{
|
||||||
options.router-settings = {
|
options.router-settings = {
|
||||||
|
routerMac = lib.mkOption {
|
||||||
|
description = "router's mac address";
|
||||||
|
type = lib.types.str;
|
||||||
|
};
|
||||||
serverMac = lib.mkOption {
|
serverMac = lib.mkOption {
|
||||||
description = "server's mac address";
|
description = "server's mac address";
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
|
|
Loading…
Reference in a new issue