From aa6fef1d9c91b2c26e7da2c701072dd7b16e1965 Mon Sep 17 00:00:00 2001 From: chayleaf Date: Wed, 9 Aug 2023 23:43:17 +0700 Subject: [PATCH] router: mss clamping also other misc changes such as fix boot on the laptop --- flake.lock | 23 ++++++------- flake.nix | 3 +- pkgs/default.nix | 11 +++++- system/hosts/nixmsi.nix | 13 ++++--- system/hosts/nixserver/home.nix | 33 +++++++++++------- system/hosts/router/default.nix | 60 +++++++++++++++++++-------------- system/modules/common.nix | 29 ++++++++++------ 7 files changed, 105 insertions(+), 67 deletions(-) diff --git a/flake.lock b/flake.lock index 7a12b04..3e33c29 100644 --- a/flake.lock +++ b/flake.lock @@ -228,11 +228,11 @@ ] }, "locked": { - "lastModified": 1690432004, - "narHash": "sha256-mGK512GjUbTNdy6AorlX6OF3oAZ5GkGWuxEKWj8xpUw=", + "lastModified": 1691308996, + "narHash": "sha256-eXxZ7Mib2U1pfcchrCqSGDRZBjCIbVIWEq93OcE6pEI=", "owner": "chayleaf", "repo": "nixos-router", - "rev": "c86131f52922907d77653d553851f03a8e064071", + "rev": "c9528e1dc5acc77273543c88db47088ea5dd28b4", "type": "github" }, "original": { @@ -243,16 +243,15 @@ }, "nixpkgs": { "locked": { - "lastModified": 1690548937, - "narHash": "sha256-x3ZOPGLvtC0/+iFAg9Kvqm/8hTAIkGjc634SqtgaXTA=", - "owner": "nixos", + "lastModified": 1691178599, + "narHash": "sha256-P4r79P2toVs0dfjXvyIp8QdnNcrFQQRFEUl/fHJBNz0=", + "owner": "chayleaf", "repo": "nixpkgs", - "rev": "2a9d660ff0f7ffde9d73be328ee6e6f10ef66b28", + "rev": "f18d5184718a901ba5b1ab437570ad291cc1873c", "type": "github" }, "original": { - "owner": "nixos", - "ref": "nixos-unstable", + "owner": "chayleaf", "repo": "nixpkgs", "type": "github" } @@ -302,11 +301,11 @@ ] }, "locked": { - "lastModified": 1689260608, - "narHash": "sha256-GHnXziKo8EzGnvMzKEExA5/H3N1Hp3GtRy84E2z4YN8=", + "lastModified": 1691303305, + "narHash": "sha256-9JUabsdRAZl4ixq4/m7+vryk4Tv4Dq3JTNAUxGV+nOU=", "owner": "chayleaf", "repo": "notnft", - "rev": "98ff79679cfdc31f49abaeb513cb6d36905dd32e", + "rev": "f7fa096c285c0705bffc180d0d08ea82f3b8e957", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 4d387b3..9047040 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,8 @@ description = "NixOS + Home Manager configuration of chayleaf"; inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + # nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:chayleaf/nixpkgs"; nixos-hardware.url = "github:NixOS/nixos-hardware"; impermanence.url = "github:nix-community/impermanence"; nur.url = "github:nix-community/NUR"; diff --git a/pkgs/default.nix b/pkgs/default.nix index 80ea878..c810745 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -27,13 +27,22 @@ in unstable = nixForNixPlugins; }); /* Various patches to change Nix version of existing packages so they don't error out because of nix-plugins in nix.conf - hydra_unstable = pkgs.hydra_unstable.override { nix = nixForNixPlugins; }; harmonia = pkgs.harmonia.override { nix = nixForNixPlugins; }; nix-init = pkgs.nix-init.override { nix = nixForNixPlugins; }; nix-serve = pkgs.nix-serve.override { nix = nixForNixPlugins; }; nix-serve-ng = pkgs.nix-serve-ng.override { nix = nixForNixPlugins; }; nurl = pkgs.nurl.override { nixVersions = builtins.mapAttrs (k: v: nixForNixPlugins) pkgs.nixVersions; }; */ + # TODO: + /*hydra_unstable = (pkgs.hydra_unstable.override { + nix = nixForNixPlugins; + }).overrideAttrs (old: { + version = "2023-05-08"; + src = old.src.override { + rev = "13ef4e3c5d87bc6f68c91a36d78cdc7d589d8ff2"; + sha256 = "sha256-niw0RHfwpo2/86wvtHrbU/DQYlkkwtrM+qG7GEC0qAo="; + }; + });*/ clang-tools_latest = pkgs.clang-tools_16; clang_latest = pkgs.clang_16; diff --git a/system/hosts/nixmsi.nix b/system/hosts/nixmsi.nix index 12048e9..8cea0d3 100644 --- a/system/hosts/nixmsi.nix +++ b/system/hosts/nixmsi.nix @@ -92,7 +92,7 @@ # System76 scheduler (not actually a scheduler, just a renice daemon) for improved responsiveness /*services.dbus.packages = [ pkgs.system76-scheduler ]; - systemd.services."system76-scheduler" = { + systemd.services.system76-scheduler = { description = "Automatically configure CPU scheduler for responsiveness on AC"; wantedBy = [ "multi-user.target" ]; serviceConfig = { @@ -150,17 +150,22 @@ services.sshd.enable = true; boot.binfmt.emulatedSystems = [ "aarch64-linux" ]; nix.settings = { - trusted-users = [ "root" config.common.mainUsername ]; netrc-file = "/secrets/netrc"; substituters = [ "https://binarycache.pavluk.org" "https://cache.nixos.org/" - # "https://nix-community.cachix.org" + ]; + trusted-substituters = [ + "https://nix-community.cachix.org" + "https://nix-gaming.cachix.org" + "https://nixpkgs-wayland.cachix.org" ]; trusted-public-keys = [ "binarycache.pavluk.org:Vk0ms/vSqoOV2JXeNVOroc8EfilgVxCCUtpCShGIKsQ=" "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" - # "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "nix-gaming.cachix.org-1:nbjlureqMbRAxR1gJ/f3hxemL9svXaZF/Ees8vCUUs4=" + "nixpkgs-wayland.cachix.org-1:3lwxaILxMRkVhehr5StQprHdEo4IrE8sRho9R9HOLYA=" ]; }; services.udev.packages = [ diff --git a/system/hosts/nixserver/home.nix b/system/hosts/nixserver/home.nix index 03d3cee..0c1798d 100644 --- a/system/hosts/nixserver/home.nix +++ b/system/hosts/nixserver/home.nix @@ -70,23 +70,25 @@ in { proxy_read_timeout 300; proxy_connect_timeout 300; proxy_send_timeout 300; + client_body_timeout 300; + send_timeout 300; ''; locations."/".proxyPass = "http://${lib.quoteListenAddr config.services.hydra.listenHost}:${toString config.services.hydra.port}/"; - locations."/static/".root = "${config.services.hydra.package}/libexec/hydra/root/"; + locations."/static/".root = lib.mkIf config.services.hydra.enable "${config.services.hydra.package}/libexec/hydra/root/"; }; users.users.nginx.extraGroups = [ "grafana" ]; - services.nix-serve = { + /*services.nix-serve = { enable = true; package = pkgs.nix-serve-ng; bindAddress = "127.0.0.1"; secretKeyFile = "/secrets/cache-priv-key.pem"; - }; - /*services.harmonia = { + };*/ + services.harmonia = { enable = true; signKeyPath = "/secrets/cache-priv-key.pem"; settings.bind = "[::1]:5000"; - };*/ + }; nix.settings.allowed-users = [ "nix-serve" "harmonia" "hydra" "hydra-www" ]; # only hydra has access to this file anyway nix.settings.extra-builtins-file = "/etc/nixos/private/extra-builtins.nix"; @@ -106,28 +108,31 @@ in { enableACME = true; forceSSL = true; basicAuthFile = "/secrets/home_password"; - locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}"; + /*locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}"; extraConfig = '' proxy_read_timeout 300; proxy_connect_timeout 300; proxy_send_timeout 300; - ''; + '';*/ # TODO: fix # https://github.com/nix-community/harmonia/issues/120 - /*locations."/".proxyPass = "http://${config.services.harmonia.settings.bind or "[::1]:5000"}"; + locations."/".proxyPass = "http://${config.services.harmonia.settings.bind or "[::1]:5000"}"; locations."/".extraConfig = '' proxy_set_header Host $host; proxy_http_version 1.1; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; - '';*/ + location ~ "^/nar/([a-z0-9]{32})-.*\.narinfo$" { + proxy_pass http://127.0.0.1:5000/$1.narinfo$is_args$args; + } + ''; # zstd on; # zstd_types application/x-nix-archive; }; services.hydra = { - enable = true; + enable = false; hydraURL = "home.${cfg.domainName}/hydra"; listenHost = "127.0.0.1"; minimumDiskFree = 30; @@ -151,9 +156,11 @@ in { systemd.services.nix-daemon.serviceConfig.CPUQuota = "100%"; nix.daemonCPUSchedPolicy = "idle"; nix.daemonIOSchedClass = "idle"; - systemd.services.hydra-evaluator.serviceConfig.CPUQuota = "100%"; - systemd.services.hydra-evaluator.serviceConfig.CPUSchedulingPolicy = "idle"; - systemd.services.hydra-evaluator.serviceConfig.IOSchedulingClass = "idle"; + systemd.services.hydra-evaluator = lib.mkIf config.services.hydra.enable { + serviceConfig.CPUQuota = "100%"; + serviceConfig.CPUSchedulingPolicy = "idle"; + serviceConfig.IOSchedulingClass = "idle"; + }; programs.ccache.enable = true; services.nginx.statusPage = true; diff --git a/system/hosts/router/default.nix b/system/hosts/router/default.nix index 39a4bc3..4ae753e 100644 --- a/system/hosts/router/default.nix +++ b/system/hosts/router/default.nix @@ -43,11 +43,16 @@ let preamble = true; country3 = "0x49"; # indoor }; + # routing tables wan_table = 1; # vpn table, assign an id but don't actually add a rule for it, so it is the default vpn_table = 2; + vpn_mtu = config.networking.wireguard.interfaces.wg0.mtu; + vpn_ipv4_mss = vpn_mtu - 40; + vpn_ipv6_mss = vpn_mtu - 60; + dnatRuleMode = rule: if rule.mode != "" then rule.mode else if rule.target4.address or null == netAddresses.lan4 || rule.target6.address or null == netAddresses.lan6 then "rule" @@ -109,9 +114,9 @@ let [(is.eq ip.protocol (f: f.icmp)) (is.ne icmp.type (f: f.echo-request)) (limit { rate = 100; per = f: f.second; }) accept] [(is.eq ip6.nexthdr (f: f.ipv6-icmp)) (is.eq icmpv6.type (f: f.echo-request)) (limit { rate = 50; per = f: f.second; }) accept] [(is.eq ip6.nexthdr (f: f.ipv6-icmp)) (is.ne icmpv6.type (f: f.echo-request)) (limit { rate = 100; per = f: f.second; }) accept] - # always accept destination unreachable and time-exceeded - [(is.eq ip.protocol (f: f.icmp)) (is.eq icmp.type (f: with f; set [ destination-unreachable time-exceeded ])) accept] - [(is.eq ip6.nexthdr (f: f.ipv6-icmp)) (is.eq icmpv6.type (f: with f; set [ destination-unreachable time-exceeded ])) accept] + # always accept destination unreachable, time-exceeded, parameter-problem, packet-too-big + [(is.eq ip.protocol (f: f.icmp)) (is.eq icmp.type (f: with f; set [ destination-unreachable time-exceeded parameter-problem ])) accept] + [(is.eq ip6.nexthdr (f: f.ipv6-icmp)) (is.eq icmpv6.type (f: with f; set [ destination-unreachable time-exceeded parameter-problem packet-too-big ])) accept] # don't log echo-request drops [(is.eq ip.protocol (f: f.icmp)) (is.eq icmp.type (f: f.echo-request)) drop] [(is.eq ip6.nexthdr (f: f.ipv6-icmp)) (is.eq icmpv6.type (f: f.echo-request)) drop] @@ -279,16 +284,16 @@ in { } ++ lib.optional (udpOnly != [ ]) { port = notnft.dsl.set udpOnly; tcp = false; udp = true; target4.address = serverAddress4; target6.address = serverAddress6; - } ++ map (range: { + } ++ lib.flip map rangesTcpAndUdp (range: { port = notnft.dsl.range range.from range.to; tcp = true; udp = true; target4.address = serverAddress4; target6.address = serverAddress6; - }) rangesTcpAndUdp ++ map (range: { + }) ++ lib.flip map rangesTcpOnly (range: { port = notnft.dsl.range range.from range.to; tcp = true; udp = false; target4.address = serverAddress4; target6.address = serverAddress6; - }) rangesTcpOnly ++ map (range: { + }) ++ lib.flip map rangesUdpOnly (range: { port = notnft.dsl.range range.from range.to; tcp = false; udp = true; target4.address = serverAddress4; target6.address = serverAddress6; - }) rangesUdpOnly; + }); router.enable = true; # 2.4g ap @@ -325,32 +330,32 @@ in { # ethernet lan0-3 router.interfaces.lan0 = { bridge = "br0"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; router.interfaces.lan1 = { bridge = "br0"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; router.interfaces.lan2 = { bridge = "br0"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; router.interfaces.lan3 = { bridge = "br0"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; # sfp lan4 router.interfaces.lan4 = { bridge = "br0"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; /* # sfp lan5 router.interfaces.lan5 = { bridge = "br0"; # i could try to figure out why this doesn't work... but i don't even have sfp to plug into this - systemdLinkMatchConfig.OriginalName = "eth1"; - systemdLinkLinkConfig.MACAddressPolicy = "persistent"; + systemdLink.matchConfig.OriginalName = "eth1"; + systemdLink.linkConfig.MACAddressPolicy = "persistent"; }; */ # ethernet wan @@ -358,8 +363,8 @@ in { dependentServices = [ { service = "wireguard-wg0"; inNetns = false; } ]; - systemdLinkLinkConfig.MACAddressPolicy = "none"; - systemdLinkLinkConfig.MACAddress = cfg.routerMac; + systemdLink.linkConfig.MACAddressPolicy = "none"; + systemdLink.linkConfig.MACAddress = cfg.routerMac; dhcpcd = { enable = true; # technically this should be assigned to br0 instead of veth-wan-b @@ -463,7 +468,7 @@ in { [ (is.eq meta.iifname "wg0") (is.eq ip.protocol protocols) (is.eq th.dport rule.port) (if rule4.port == null then dnat.ip rule4.address else dnat.ip rule4.address rule4.port) ] ] ++ lib.optionals (rule6 != null) [ - [ (is.eq meta.iifname "wg0") (is.eq ip6.protocol protocols) (is.eq th.dport rule.port) + [ (is.eq meta.iifname "wg0") (is.eq ip6.nexthdr protocols) (is.eq th.dport rule.port) (if rule6.port == null then dnat.ip6 rule6.address else dnat.ip6 rule6.address rule6.port) ] ]) (builtins.filter (x: x.inVpn && (x.tcp || x.udp)) cfg.dnatRules)); @@ -500,15 +505,17 @@ in { prerouting = add chain { type = f: f.filter; hook = f: f.prerouting; prio = f: f.filter; policy = f: f.accept; } ([ [(mangle meta.mark ct.mark)] [(is.ne meta.mark 0) accept] + [(is.eq ip.daddr "@block4") drop] + [(is.eq ip6.daddr "@block6") drop] [(is.eq meta.iifname "br0") (mangle meta.mark vpn_table)] [(is.eq ip.daddr "@force_unvpn4") (mangle meta.mark wan_table)] [(is.eq ip6.daddr "@force_unvpn6") (mangle meta.mark wan_table)] - # don't vpn smtp requests so spf works fine (and in case the vpn blocks requests over port 25) - [(is.eq ip.saddr serverAddress4) (is.eq ip.protocol (f: f.tcp)) (is.eq tcp.dport 25) (mangle meta.mark wan_table)] - [(is.eq ip6.saddr serverAddress6) (is.eq ip6.nexthdr (f: f.tcp)) (is.eq tcp.dport 25) (mangle meta.mark wan_table)] - # but block requests to port 25 from other hosts so they can't send mail pretending to originate from my domain + # block requests to port 25 from hosts other than the server so they can't send mail pretending to originate from my domain [(is.ne ip.saddr serverAddress4) (is.eq ip.protocol (f: f.tcp)) (is.eq tcp.dport 25) drop] [(is.ne ip6.saddr serverAddress6) (is.eq ip6.nexthdr (f: f.tcp)) (is.eq tcp.dport 25) drop] + # don't vpn smtp requests so spf works fine (and in case the vpn blocks requests over port 25, which it usually does) + [(is.eq ip.protocol (f: f.tcp)) (is.eq tcp.dport 25) (mangle meta.mark wan_table)] + [(is.eq ip6.nexthdr (f: f.tcp)) (is.eq tcp.dport 25) (mangle meta.mark wan_table)] [(is.eq ip.daddr "@force_vpn4") (mangle meta.mark vpn_table)] [(is.eq ip6.daddr "@force_vpn6") (mangle meta.mark vpn_table)] ] ++ # 1. dnat non-vpn: change rttable to wan @@ -537,19 +544,22 @@ in { [ (is ct.status (f: f.dnat)) (is.eq meta.iifname "br0") (is.eq ip.protocol protocols) (is.eq ip.saddr rule4.address) (is.eq th.sport (if rule4.port != null then rule4.port else rule.port)) (mangle meta.mark vpn_table) ] ] ++ lib.optionals (rule6 != null) [ - [ (is ct.status (f: f.dnat)) (is.eq meta.iifname "br0") (is.eq ip6.protocol protocols) (is.eq ip6.saddr rule6.address) + [ (is ct.status (f: f.dnat)) (is.eq meta.iifname "br0") (is.eq ip6.nexthdr protocols) (is.eq ip6.saddr rule6.address) (is.eq th.sport (if rule6.port != null then rule6.port else rule.port)) (mangle meta.mark vpn_table) ] ]) (builtins.filter (x: x.inVpn && (x.tcp || x.udp) && dnatRuleMode x == "mark") cfg.dnatRules)) ++ [ - [(is.eq ip.daddr "@block4") drop] - [(is.eq ip6.daddr "@block6") drop] # this doesn't work... it still gets routed, even though iot_table doesn't have a default route - # [(is.eq ip.saddr vacuumAddress4) (is.ne ip.daddr) (mangle meta.mark iot_table)] # [(is.eq ether.saddr cfg.vacuumMac) (mangle meta.mark iot_table)] # instead of debugging that, simply change the approach [(is.eq ether.saddr cfg.vacuumMac) (is.ne ip.daddr (cidr netCidrs.lan4)) (is.ne ip.daddr "@allow_iot4") (log "iot4 ") drop] [(is.eq ether.saddr cfg.vacuumMac) (is.ne ip6.daddr (cidr netCidrs.lan6)) (is.ne ip6.daddr "@allow_iot6") (log "iot6 ") drop] + # MSS clamping - since VPN reduces max MTU + # We only do this for the first packet in a connection, which should be enough + [(is.eq ip6.nexthdr (f: f.tcp)) (is.eq meta.mark vpn_table) + (is.gt tcpOpt.maxseg.size vpn_ipv6_mss) (mangle tcpOpt.maxseg.size vpn_ipv6_mss)] + [(is.eq ip.protocol (f: f.tcp)) (is.eq meta.mark vpn_table) + (is.gt tcpOpt.maxseg.size vpn_ipv4_mss) (mangle tcpOpt.maxseg.size vpn_ipv4_mss)] [(mangle ct.mark meta.mark)] ]); }; diff --git a/system/modules/common.nix b/system/modules/common.nix index c70e54a..967279b 100644 --- a/system/modules/common.nix +++ b/system/modules/common.nix @@ -3,7 +3,10 @@ , config , ... }: -{ +let + # force some defaults even if they were set with mkDefault already... + mkForceDefault = lib.mkOverride 999; +in { options.common = with lib; mkOption { type = types.submodule { options = { @@ -35,9 +38,18 @@ cfg = config.common; in { nix = { + channel.enable = false; settings = { allowed-users = [ cfg.mainUsername ]; auto-optimise-store = true; + use-xdg-base-directories = true; + experimental-features = [ + "ca-derivations" + "flakes" + "nix-command" + "no-url-literals" + "repl-flake" + ]; }; gc = { automatic = true; @@ -45,9 +57,6 @@ options = "--delete-older-than 30d"; }; package = pkgs.nixForNixPlugins; - extraOptions = '' - experimental-features = nix-command flakes - ''; }; systemd.services.nix-daemon.serviceConfig.LimitSTACKSoft = "infinity"; boot.kernelParams = lib.optionals (cfg.resolution != null) [ @@ -105,23 +114,21 @@ }; }; # this is supposed to default to false, but it doesn't because of nixos fish module - documentation.man.generateCaches = lib.mkOverride 999 false; - # and we don't need html files and so on on minimal machines (it's not like I ever use it anyway) - # as a bonus, this disables the HTML NixOS manual which takes a while to build and which I - # definitely don't need on minimal machines + documentation.man.generateCaches = lib.mkIf cfg.minimal (mkForceDefault false); + # we don't need stuff like html files (NixOS manual and so on) on minimal machines documentation.doc.enable = lib.mkIf cfg.minimal (lib.mkDefault false); programs.fish.enable = true; # conflicts with bash module's mkDefault # only override on minimal systems because on non-minimal systems - # my fish config doesn't work well in fb/drm console - users.defaultUserShell = lib.mkIf cfg.minimal (lib.mkOverride 999 pkgs.fish); + # because my fish config doesn't work well in fb/drm console + users.defaultUserShell = lib.mkIf cfg.minimal (mkForceDefault pkgs.fish); users.users.${cfg.mainUsername} = { uid = 1000; isNormalUser = true; extraGroups = [ "wheel" ]; }; # nixos-hardware uses mkDefault here, so we use slightly higher priority - services.xserver.libinput.enable = lib.mkOverride 999 (!cfg.minimal); + services.xserver.libinput.enable = mkForceDefault (!cfg.minimal); # TODO: minimal fish/vim config /* services.xserver = {