router: final fixes, it's now in production

This commit is contained in:
chayleaf 2023-06-24 09:49:57 +07:00
parent 8894e0d89c
commit 561a481f1a
8 changed files with 173 additions and 148 deletions

View file

@ -43,6 +43,8 @@
outputs = inputs@{ self, nixpkgs, nixos-hardware, impermanence, home-manager, nur, nix-gaming, notlua, notnft, nixos-mailserver, nixos-router, ... }:
let
# --impure required for developing
# it takes the paths for notlua,notnft,nixos-router from filesystem as opposed to flake inputs
developing = false;
# IRL-related stuff I'd rather not put into git
priv =
@ -80,7 +82,7 @@
mkPkgs = config: import nixpkgs (config // {
overlays = (if config?overlays then config.overlays else [ ]) ++ [ overlay ];
});
# this is actual config, it gets processed later
# this is actual config, it gets processed below
config = {
nixserver = {
modules = [
@ -93,6 +95,7 @@
system = "aarch64-linux";
specialArgs.notnft = if developing then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if developing then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.server-config = nixosConfigurations.nixserver.config;
modules = [
./system/hardware/bpi_r3/emmc.nix
./system/hosts/router
@ -104,6 +107,7 @@
system = "aarch64-linux";
specialArgs.notnft = if developing then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if developing then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.server-config = nixosConfigurations.nixserver.config;
modules = [
./system/hardware/bpi_r3/sd.nix
./system/hosts/router
@ -128,25 +132,8 @@
];
};
};
in rec {
overlays.default = overlay;
packages = lib.genAttrs [
"x86_64-linux"
"aarch64-linux"
] (system: let self = overlay self (import nixpkgs { inherit system; }); in self );
nixosImages.router = let pkgs = import nixpkgs { system = "aarch64-linux"; overlays = [ overlay ]; }; in {
emmcImage = pkgs.callPackage ./system/hardware/bpi_r3/image.nix {
inherit (nixosConfigurations.router-emmc) config;
rootfsImage = nixosConfigurations.router-emmc.config.system.build.rootfsImage;
bpiR3Stuff = pkgs.bpiR3StuffEmmc;
};
sdImage = pkgs.callPackage ./system/hardware/bpi_r3/image.nix {
inherit (nixosConfigurations.router-sd) config;
rootfsImage = nixosConfigurations.router-sd.config.system.build.rootfsImage;
bpiR3Stuff = pkgs.bpiR3StuffSd;
};
};
# this is the system config part
# this is the system config processing part
nixosConfigurations = builtins.mapAttrs (hostname: args @ { system ? "x86_64-linux", modules, specialArgs ? {}, nixpkgs ? {}, home ? {}, ... }:
lib.nixosSystem ({
inherit system;
@ -221,6 +208,25 @@
]);
} // (builtins.removeAttrs args [ "home" "modules" "nixpkgs" ])))
config;
in {
inherit nixosConfigurations;
overlays.default = overlay;
packages = lib.genAttrs [
"x86_64-linux"
"aarch64-linux"
] (system: let self = overlay self (import nixpkgs { inherit system; }); in self );
nixosImages.router = let pkgs = import nixpkgs { system = "aarch64-linux"; overlays = [ overlay ]; }; in {
emmcImage = pkgs.callPackage ./system/hardware/bpi_r3/image.nix {
inherit (nixosConfigurations.router-emmc) config;
rootfsImage = nixosConfigurations.router-emmc.config.system.build.rootfsImage;
bpiR3Stuff = pkgs.bpiR3StuffEmmc;
};
sdImage = pkgs.callPackage ./system/hardware/bpi_r3/image.nix {
inherit (nixosConfigurations.router-sd) config;
rootfsImage = nixosConfigurations.router-sd.config.system.build.rootfsImage;
bpiR3Stuff = pkgs.bpiR3StuffSd;
};
};
# for each hostname, for each user, generate an attribute "${user}@${hostname}"
homeConfigurations =

View file

@ -2,14 +2,11 @@
, e2fsprogs
, util-linux
, zstd
, stdenvNoCC
, btrfs-progs
, vmTools
, runCommand
, jq
, kmod
, udev
, nix
, lkl
, bpiR3Stuff
, config
@ -17,7 +14,7 @@
, ... }:
let
imageSize = 7818182656;
imageSize = 7818182656; # emmc size
bl2Start = 34;
bl2End = 8191;
envStart = 8192;
@ -33,26 +30,44 @@ in
# the vm is suuuuuper slow
# so, do as much as possible outside of it
# but i still need it to create subvolumes
let
template = vmTools.runInLinuxVM (runCommand "bpi-r3-fs-template" {
preVM = ''
truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
${btrfs-progs}/bin/mkfs.btrfs \
--label NIXOS_SD \
--uuid "44444444-4444-4444-8888-888888888888" \
./tmp.img
'';
nativeBuildInputs = [ btrfs-progs e2fsprogs util-linux kmod ];
postVM = ''
mkdir -p $out
${zstd}/bin/zstd tmp.img -o $out/template.btrfs.zst
'';
memSize = "4G";
QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
} ''
modprobe btrfs
mkdir -p /mnt
mount -t btrfs -o space_cache=v2 /dev/vda /mnt
btrfs filesystem resize max /mnt
btrfs subvolume create /mnt/@boot
btrfs subvolume create /mnt/@
btrfs subvolume create /mnt/@nix
chattr +C /mnt/@boot
'');
in
vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
enableParallelBuildingByDefault = true;
stdenvNoCC.mkDerivation {
name = "bpi-r3-fs";
nativeBuildInputs = [
btrfs-progs
# dosfstools
e2fsprogs
util-linux # sfdisk
config.system.build.nixos-install
jq
kmod
udev
gptfdisk
zstd
nix
lkl
];
preVM = ''
# ${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
unpackPhase = "true";
buildPhase = ''
img=./result.img
${vmTools.qemu}/bin/qemu-img create -f raw $img 7818182656
truncate -s ${toString imageSize} $img
${gptfdisk}/bin/sgdisk -o \
@ -65,34 +80,17 @@ vmTools.runInLinuxVM (runCommand "bpi-r3-fs" {
$img
dd conv=notrunc if=${bpiR3Stuff}/bl2.img of=$img seek=${toString bl2Start}
dd conv=notrunc if=${bpiR3Stuff}/fip.bin of=$img seek=${toString fipStart}
truncate -s ${toString ((rootPartEnd - rootPartStart + 1) * 512)} ./tmp.img
${btrfs-progs}/bin/mkfs.btrfs \
--label NIXOS_SD \
--uuid "44444444-4444-4444-8888-888888888888" \
./tmp.img
'';
# i give up on making this work in a nix derivation, just do the rest in a script (./image.sh)
# (i could do more in a vm but thats too slow, i could program a custom userspace utility but thats too annoying)
postVM = ''
# Truncate the file at the end of the last partition
img=./result.img
installPhase = ''
mkdir -p $out/boot
${config.boot.loader.generic-extlinux-compatible.populateCmd} -c ${config.system.build.toplevel} -d $out/boot -g 0
cp ${rootfsImage} $out/rootfs.ext4
cp tmp.img $out/template.btrfs
cp $img $out/image.img
ln -s ${rootfsImage} $out/rootfs.ext4
cp ${template}/template.btrfs.zst $out
zstd $img -o $out/image.img.zst
echo ${toString rootPartStart} > $out/metadata
echo ${toString rootPartEnd} >> $out/metadata
ln -s ${bpiR3Stuff}/bl2.img $out/boot0.img
'';
memSize = "4G";
QEMU_OPTS = "-drive file=./tmp.img,format=raw,if=virtio,cache=unsafe,werror=report -rtc base=2000-01-01,clock=vm";
} ''
modprobe btrfs
mkdir -p /mnt
mount -t btrfs -o space_cache=v2 /dev/vda /mnt
btrfs filesystem resize max /mnt
btrfs subvolume create /mnt/@boot
btrfs subvolume create /mnt/@
btrfs subvolume create /mnt/@nix
chattr +C /mnt/@boot
'')
}

View file

@ -46,8 +46,8 @@ if [ -z "$1" ]; then
exit 1
fi
rootfs="$1/rootfs.ext4"
template="$1/template.btrfs"
image="$1/image.img"
template="$1/template.btrfs.zst"
image="$1/image.img.zst"
boot="$1/boot"
metadata0="$(head -n1 "$1/metadata")"
@ -58,10 +58,12 @@ if [ -z "$metadata0" ] || [ -z "$metadata1" ] || [[ "$metadata0" == "$metadata1"
fi
tmp=$(mktemp -d)
cp "$template" "$tmp/template.btrfs"
cp "$image" "$tmp/image.img"
cp "$template" "$tmp/template.btrfs.zst"
cp "$image" "$tmp/image.img.zst"
template="$tmp/template.btrfs"
image="$tmp/image.img"
unzstd --rm "$template.zst"
unzstd --rm "$image.zst"
chmod +w "$template" "$image"
function cleanup {
@ -94,5 +96,4 @@ run umount "$tmp/out"
dd conv=notrunc if="$template" of="$image" seek="$metadata0"
zstd --compress "$image"
cp "$image.zst" ./image.img.zst
zstd -f --rm --compress "$image" -o ./image.img.zst

View file

@ -85,9 +85,6 @@ in {
};
console.font = "${pkgs.terminus_font}/share/consolefonts/ter-v24n.psf.gz";
networking.useDHCP = true;
networking.resolvconf.extraConfig = ''
name_servers="127.0.0.1 ::1"
'';
networking.firewall = {
enable = true;
allowedTCPPorts = [
@ -107,12 +104,15 @@ in {
# UNBOUND
users.users.${config.common.mainUsername}.extraGroups = [ config.services.unbound.group ];
#networking.resolvconf.extraConfig = ''
# name_servers="127.0.0.1 ::1"
#'';
services.unbound = {
enable = true;
enable = false;
package = pkgs.unbound-with-systemd.override {
stdenv = pkgs.ccacheStdenv;
withPythonModule = true;
python = unbound-python;
python = pkgs.python3;
};
localControlSocketPath = "/run/unbound/unbound.ctl";
resolveLocalQueries = false;
@ -142,9 +142,11 @@ in {
remote-control.control-enable = true;
};
};
systemd.services.unbound.environment = {
MDNS_ACCEPT_NAMES = "^.*\\.local\\.$";
PYTHONPATH = "${unbound-python}/${unbound-python.sitePackages}";
systemd.services.unbound = lib.mkIf config.services.unbound.enable {
environment = {
MDNS_ACCEPT_NAMES = "^.*\\.local\\.$";
PYTHONPATH = "${unbound-python}/${unbound-python.sitePackages}";
};
};
# just in case
networking.hosts."127.0.0.1" = [ "localhost" ] ++ hosted-domains;
@ -179,6 +181,7 @@ in {
enable = true;
ignoreIP = lib.optionals (cfg.lanCidrV4 != "0.0.0.0/0") [ cfg.lanCidrV4 ]
++ (lib.optionals (cfg.lanCidrV6 != "::/0") [ cfg.lanCidrV6 ]);
maxretry = 10;
jails.dovecot = ''
enabled = true
filter = dovecot

View file

@ -681,7 +681,9 @@ def operate(id, event, qstate, qdata):
f.write(new_data)
if changed4:
for qname in qnames:
name4 = NFT_QUERIES[qname]['name4']
q = NFT_QUERIES[qname]
name4 = q['name4']
ips4 = q['ips4']
if name4:
ip2 = []
for ip in ip4:
@ -698,7 +700,9 @@ def operate(id, event, qstate, qdata):
add_ips(name4, False, ip2)
if changed6:
for qname in qnames:
name6 = NFT_QUERIES[qname]['name6']
q = NFT_QUERIES[qname]
name6 = q['name6']
ips6 = q['ips6']
if name6:
ip2 = []
for ip in ip6:

View file

@ -3,6 +3,7 @@
, notnft
, lib
, router-lib
, server-config
, ... }:
let
@ -98,7 +99,8 @@ let
[return];
ingress_lan_common = add chain
[(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
# there are some issues with this, disable it for lan
# [(is.eq (fib (f: with f; [ saddr mark iif ]) (f: f.oif)) missing) (log "${logPrefix}oif missing ") drop]
[(jump "ingress_common")];
ingress_wan_common = add chain
@ -151,8 +153,8 @@ let
}
// lib.genAttrs lans (_: jump "inbound_lan_common")
// lib.genAttrs wans (_: jump "inbound_wan_common")
))]
[(log "${logPrefix}inbound drop ")];
))];
#[(log "${logPrefix}inbound drop ")];
forward = add chain { type = f: f.filter; hook = f: f.forward; prio = f: f.filter; policy = f: f.drop; }
[(vmap ct.state { established = accept; related = accept; invalid = drop; })]
@ -215,14 +217,7 @@ let
vacuumAddress4 = addToIp parsedGatewayAddr4 2;
vacuumAddress6 = addToIp parsedGatewayAddr6 2;
# TODO: take from server config?
hosted-domains =
map
(prefix: if prefix == null then cfg.domainName else "${prefix}.${cfg.domainName}")
[
null "dns" "mumble" "mail" "music" "www" "matrix"
"search" "git" "cloud" "ns1" "ns2"
];
hosted-domains = builtins.attrNames server-config.services.nginx.virtualHosts;
in {
router-settings.domainName = "pavluk.org";
router-settings.dhcpReservations = [
@ -237,34 +232,40 @@ in {
{ ipAddress = vacuumAddress6;
macAddress = cfg.vacuumMac; }
];
router-settings.dnatRules = [
{
# TODO: take firewall settings from server config
port = notnft.dsl.set [
# http
80 443 8008 8448
# mail
25 587 465 143 993 110 995 4190
];
tcp = true; udp = false;
target4.address = serverAddress4;
target6.address = serverAddress6;
}
{
# mumble
port = 64738; tcp = true; udp = true;
target4.address = serverAddress4;
target6.address = serverAddress6;
}
{
# expose the default namespace's ssh via port 23
port = 23; tcp = true; udp = false;
target4.address = gatewayAddr4;
target4.port = 22;
target6.address = gatewayAddr6;
target6.port = 22;
}
];
# dnat to server, take ports from its firewall config
router-settings.dnatRules = let
allTcp = server-config.networking.firewall.allowedTCPPorts;
allTcpRanges = server-config.networking.firewall.allowedTCPPortRanges;
allUdp = server-config.networking.firewall.allowedUDPPorts;
allUdpRanges = server-config.networking.firewall.allowedUDPPortRanges;
tcpAndUdp = builtins.filter (x: x != 22 && builtins.elem x allTcp) allUdp;
tcpOnly = builtins.filter (x: x != 22 && !(builtins.elem x allUdp)) allTcp;
udpOnly = builtins.filter (x: x != 22 && !(builtins.elem x allTcp)) allUdp;
rangesTcpAndUdp = builtins.filter (x: builtins.elem x allTcpRanges) allUdpRanges;
rangesTcpOnly = builtins.filter (x: !(builtins.elem x allUdpRanges)) allTcpRanges;
rangesUdpOnly = builtins.filter (x: !(builtins.elem x allTcpRanges)) allUdpRanges;
in lib.optional (tcpAndUdp != [ ]) {
port = notnft.dsl.set tcpAndUdp; tcp = true; udp = true;
target4.address = serverAddress4; target6.address = serverAddress6;
} ++ lib.optional (tcpOnly != [ ]) {
port = notnft.dsl.set tcpOnly; tcp = true; udp = false;
target4.address = serverAddress4; target6.address = serverAddress6;
} ++ lib.optional (udpOnly != [ ]) {
port = notnft.dsl.set udpOnly; tcp = false; udp = true;
target4.address = serverAddress4; target6.address = serverAddress6;
} ++ map (range: {
port = notnft.dsl.range range.from range.to; tcp = true; udp = true;
target4.address = serverAddress4; target6.address = serverAddress6;
}) rangesTcpAndUdp ++ map (range: {
port = notnft.dsl.range range.from range.to; tcp = true; udp = false;
target4.address = serverAddress4; target6.address = serverAddress6;
}) rangesTcpOnly ++ map (range: {
port = notnft.dsl.range range.from range.to; tcp = false; udp = true;
target4.address = serverAddress4; target6.address = serverAddress6;
}) rangesUdpOnly;
imports = [ ./options.nix ];
system.stateVersion = "22.11";
@ -301,9 +302,11 @@ in {
"net.ipv6.conf.all.forwarding" = true;
};
services.openssh.enable = true;
/*services.fail2ban = {
services.fail2ban = {
enable = true;
};*/
ignoreIP = [ netCidr4 netCidr6 ];
maxretry = 10;
};
router.enable = true;
router.interfaces.wlan0 = {
bridge = "br0";
@ -364,7 +367,7 @@ in {
{ service = "wireguard-wg0"; inNetns = false; }
];
systemdLinkLinkConfig.MACAddressPolicy = "none";
systemdLinkLinkConfig.MACAddress = "11:22:33:44:55:66";
systemdLinkLinkConfig.MACAddress = cfg.routerMac;
dhcpcd.enable = true;
networkNamespace = "wan";
};
@ -408,6 +411,34 @@ in {
ipv4.kea.enable = true;
ipv6.radvd.enable = true;
ipv6.kea.enable = true;
};
router.networkNamespaces.default = {
# set routing table depending on packet mark
rules = [
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
# don't add vpn_table as it should be the default
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
] ++ builtins.concatLists (map (rule: let
table = if rule.inVpn then 0 else wan_table;
forEachPort = func: port:
if builtins.isInt port then [ (func port) ]
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
else throw "Unsupported expr: ${builtins.toJSON port}";
gen = len: proto: tgt:
forEachPort
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
(if tgt.port == null then rule.port else tgt.port);
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
nftables.stopJsonRules = mkFlushRules {};
nftables.jsonRules = mkRules {
selfIp4 = gatewayAddr4;
@ -541,6 +572,8 @@ in {
ipv6.routes = [
{ extraArgs = [ netCidr6 "via" mainNetnsAddr6 ]; }
];
};
router.networkNamespaces.wan = {
nftables.stopJsonRules = mkFlushRules {};
nftables.jsonRules = mkRules {
selfIp4 = wanNetnsAddr4;
@ -600,30 +633,6 @@ in {
];
};
};
router.networkNamespaces.default.rules = [
{ ipv6 = false; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" wan_table "table" wan_table ]; }
# don't add vpn_table as it should be the default
{ ipv6 = false; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
{ ipv6 = true; extraArgs = [ "fwmark" iot_table "table" iot_table ]; }
] ++ builtins.concatLists (map (rule: let
table = if rule.inVpn then 0 else wan_table;
forEachPort = func: port:
if builtins.isInt port then [ (func port) ]
else if port?set then builtins.concatLists (map (forEachPort func) port.set)
else if port?range.min then let inherit (port.range) min max; in [ (func "${toString min}-${toString max}") ]
else if port?range then let max = builtins.elemAt port.range 1; min = builtins.head port.range; in [ (func "${toString min}-${toString max}" ) ]
else throw "Unsupported expr: ${builtins.toJSON port}";
gen = len: proto: tgt:
forEachPort
(port: [ "from" "${tgt.address}/${toString len}" "ipproto" proto "sport" port "table" table ])
(if tgt.port == null then rule.port else tgt.port);
in lib.optionals (rule.tcp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "tcp" rule.target4))
++ lib.optionals (rule.udp && rule.target4 != null) (map (x: { ipv6 = false; extraArgs = x; }) (gen 32 "udp" rule.target4))
++ lib.optionals (rule.tcp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "tcp" rule.target6))
++ lib.optionals (rule.udp && rule.target6 != null) (map (x: { ipv6 = true; extraArgs = x; }) (gen 128 "udp" rule.target6))
) (builtins.filter (x: (x.tcp || x.udp) && dnatRuleMode x == "rule") cfg.dnatRules));
networking.wireguard.interfaces.wg0 = cfg.wireguard // {
socketNamespace = "wan";
@ -673,9 +682,9 @@ in {
# load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
# resolve domains and append it to ips and add it to the nftables sets
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6";
# it needs to run after br0 has been set up because it sets up the sets
after = [ "nftables-br0.service" ];
wants = [ "nftables-br0.service" ];
# it needs to run after nftables has been set up because it sets up the sets
after = [ "nftables-default.service" ];
wants = [ "nftables-default.service" ];
# allow it to call nft
serviceConfig.AmbientCapabilities = [ "CAP_NET_ADMIN" ];
};

View file

@ -5,6 +5,10 @@
{
options.router-settings = {
routerMac = lib.mkOption {
description = "router's mac address";
type = lib.types.str;
};
serverMac = lib.mkOption {
description = "server's mac address";
type = lib.types.str;

View file

@ -93,7 +93,7 @@ in {
] ++ lib.optionals config.services.printing.enable [
{ directory = /var/lib/cups; user = "root"; group = "root"; mode = "0755"; }
] ++ lib.optionals config.services.fail2ban.enable [
{ directory = /var/lib/fail2ban; user = "fail2ban"; group = "fail2ban"; mode = "0750"; }
{ directory = /var/lib/fail2ban; user = "root"; group = "root"; mode = "0700"; }
] ++ lib.optionals config.services.opendkim.enable [
{ directory = /var/lib/opendkim; user = "opendkim"; group = "opendkim"; mode = "0700"; }
] ++ lib.optionals config.services.pleroma.enable [