server: btrfs->bcachefs; add maubot.nix; update searxng

router: add remote query editing support
This commit is contained in:
chayleaf 2023-06-27 15:25:19 +07:00
parent 5211eb8d71
commit 9d96dc1a44
14 changed files with 226 additions and 69 deletions

View file

@ -33,6 +33,22 @@
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_3": {
"flake": false,
"locked": {
"lastModified": 1668681692,
@ -67,6 +83,21 @@
}
},
"flake-utils": {
"locked": {
"lastModified": 1678901627,
"narHash": "sha256-U02riOqrKKzwjsxc/400XnElV+UtPUQWpANPlyazjH0=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "93a2b84fc4b70d9e089d029deacc3583435c2ed6",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems"
},
@ -119,6 +150,28 @@
"type": "github"
}
},
"maubot": {
"inputs": {
"flake-compat": "flake-compat_2",
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1687853065,
"narHash": "sha256-HNq95YrJm8ng7lSdGbyDCihgrS6xhQm6Agyej6ttmGg=",
"owner": "chayleaf",
"repo": "maubot.nix",
"rev": "f06cffda880a0a403a3b4c40263a03dd2523775b",
"type": "github"
},
"original": {
"owner": "chayleaf",
"repo": "maubot.nix",
"type": "github"
}
},
"nix-gaming": {
"inputs": {
"flake-parts": "flake-parts",
@ -158,7 +211,7 @@
"nixos-mailserver": {
"inputs": {
"blobs": "blobs",
"flake-compat": "flake-compat_2",
"flake-compat": "flake-compat_3",
"nixpkgs": [
"nixpkgs"
],
@ -295,6 +348,7 @@
"flake-compat": "flake-compat",
"home-manager": "home-manager",
"impermanence": "impermanence",
"maubot": "maubot",
"nix-gaming": "nix-gaming",
"nixos-hardware": "nixos-hardware",
"nixos-mailserver": "nixos-mailserver",
@ -308,7 +362,7 @@
},
"rust-overlay": {
"inputs": {
"flake-utils": "flake-utils",
"flake-utils": "flake-utils_2",
"nixpkgs": [
"nixpkgs"
]

View file

@ -30,6 +30,10 @@
url = "github:chayleaf/nixos-router";
inputs.nixpkgs.follows = "nixpkgs";
};
maubot = {
url = "github:chayleaf/maubot.nix";
inputs.nixpkgs.follows = "nixpkgs";
};
nixos-mailserver = {
url = "gitlab:simple-nixos-mailserver/nixos-mailserver";
inputs.nixpkgs.follows = "nixpkgs";
@ -41,11 +45,13 @@
};
};
outputs = inputs@{ self, nixpkgs, nixos-hardware, impermanence, home-manager, nur, nix-gaming, notlua, notnft, nixos-mailserver, nixos-router, ... }:
outputs = inputs@{ self, nixpkgs, nixos-hardware, impermanence, home-manager, nur, nix-gaming, notlua, notnft, nixos-mailserver, nixos-router, maubot, ... }:
let
# --impure required for developing
# it takes the paths for notlua,notnft,nixos-router from filesystem as opposed to flake inputs
developing = false;
# it takes the paths for modules from filesystem as opposed to flake inputs
devNft = false;
devNixRt = false;
devMaubot = false;
# IRL-related stuff I'd rather not put into git
priv =
if builtins.pathExists ./private.nix then (import ./private.nix { })
@ -88,26 +94,27 @@
modules = [
nixos-mailserver.nixosModules.default
./system/devices/hp-probook-g0-server.nix
(if devMaubot then import /${devPath}/maubot.nix/module else maubot.nixosModules.default)
];
};
router-emmc = rec {
system = "aarch64-linux";
specialArgs.notnft = if developing then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if developing then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.notnft = if devNft then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if devNixRt then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.server-config = nixosConfigurations.nixserver.config;
modules = [
(import ./system/devices/bpi-r3-router.nix "emmc")
(if developing then (import /${devPath}/nixos-router) else nixos-router.nixosModules.default)
(if devNixRt then (import /${devPath}/nixos-router) else nixos-router.nixosModules.default)
];
};
router-sd = rec {
system = "aarch64-linux";
specialArgs.notnft = if developing then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if developing then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.notnft = if devNft then (import /${devPath}/notnft { inherit (nixpkgs) lib; }).config.notnft else notnft.lib.${system};
specialArgs.router-lib = if devNixRt then import /${devPath}/nixos-router/lib.nix { inherit (nixpkgs) lib; } else nixos-router.lib.${system};
specialArgs.server-config = nixosConfigurations.nixserver.config;
modules = [
(import ./system/devices/bpi-r3-router.nix "sd")
(if developing then (import /${devPath}/nixos-router) else nixos-router.nixosModules.default)
(if devNixRt then (import /${devPath}/nixos-router) else nixos-router.nixosModules.default)
];
};
nixmsi = rec {

View file

@ -1,4 +1,18 @@
{
"atf-bpir3": {
"cargoLocks": null,
"date": "2022-12-13",
"extract": null,
"name": "atf-bpir3",
"passthru": null,
"pinned": false,
"src": {
"sha256": "sha256-pW2yytXRIFEIbG1gnuXq8TiLe/Eew7zESe6Pijh2qVk=",
"type": "tarball",
"url": "https://github.com/frank-w/u-boot/archive/c30a1caf8274af67bf31f3fb5abc45df5737df36.tar.gz"
},
"version": "c30a1caf8274af67bf31f3fb5abc45df5737df36"
},
"fastforward": {
"cargoLocks": null,
"date": null,
@ -29,6 +43,20 @@
},
"version": "GE-Proton8-4"
},
"searxng": {
"cargoLocks": null,
"date": "2023-06-25",
"extract": null,
"name": "searxng",
"passthru": null,
"pinned": false,
"src": {
"sha256": "sha256-sk28RG9/ZoPL71x99tNi884Mw0taMTYWh6HXINTr1xQ=",
"type": "tarball",
"url": "https://github.com/searxng/searxng/archive/e8706fb738da9feb21e596f403dddb40e69c8a7b.tar.gz"
},
"version": "e8706fb738da9feb21e596f403dddb40e69c8a7b"
},
"yomichan": {
"cargoLocks": null,
"date": null,

View file

@ -1,6 +1,15 @@
# This file was generated by nvfetcher, please do not modify it manually.
{ fetchgit, fetchurl, fetchFromGitHub, dockerTools }:
{
atf-bpir3 = {
pname = "atf-bpir3";
version = "c30a1caf8274af67bf31f3fb5abc45df5737df36";
src = fetchTarball {
url = "https://github.com/frank-w/u-boot/archive/c30a1caf8274af67bf31f3fb5abc45df5737df36.tar.gz";
sha256 = "sha256-pW2yytXRIFEIbG1gnuXq8TiLe/Eew7zESe6Pijh2qVk=";
};
date = "2022-12-13";
};
fastforward = {
pname = "fastforward";
version = "0.2237";
@ -17,12 +26,13 @@
sha256 = "sha256-OPwmVxBGaWo51pDJcqvxvZ8qxMH8X0DwZTpwiKbdx/I=";
};
};
yomichan = {
pname = "yomichan";
version = "22.10.23.0";
src = fetchurl {
url = "https://github.com/FooSoft/yomichan/releases/download/22.10.23.0/yomichan-firefox-dev.xpi";
sha256 = "sha256-l70wVXHEArifukDelZeoVxIyP2Crs6QZSD/kFdEml/8=";
searxng = {
pname = "searxng";
version = "e8706fb738da9feb21e596f403dddb40e69c8a7b";
src = fetchTarball {
url = "https://github.com/searxng/searxng/archive/e8706fb738da9feb21e596f403dddb40e69c8a7b.tar.gz";
sha256 = "sha256-sk28RG9/ZoPL71x99tNi884Mw0taMTYWh6HXINTr1xQ=";
};
date = "2023-06-25";
};
}

View file

@ -41,6 +41,13 @@ in
'';
};
rofi-steam-game-list = callPackage ./rofi-steam-game-list { };
searxng = pkgs.searxng.overrideAttrs (old: {
inherit (sources.searxng) src;
version = "unstable-" + sources.searxng.date;
propagatedBuildInputs = old.propagatedBuildInputs ++ (with pkgs'.python3.pkgs; [
pytomlpp
]);
});
# system76-scheduler = callPackage ./system76-scheduler.nix { };
techmino = callPackage ./techmino { };

View file

@ -22,11 +22,14 @@ in
inherit lib stdenv fetchurl;
inherit (nur.repos.rycee.firefox-addons) buildFirefoxXpiAddon;
}) // {
# addons.mozilla.org's version is horribly outdated for whatever reason
# I guess the extension normally autoupdates by itself?
# this is an unsigned build
# this is no longer maintained, hardcode last released version
yomichan = buildExtension {
inherit (sources.yomichan) pname version src;
pname = "yomichan";
version = "22.10.23.0";
src = fetchurl {
url = "https://github.com/FooSoft/yomichan/releases/download/22.10.23.0/yomichan-firefox-dev.xpi";
sha256 = "sha256-l70wVXHEArifukDelZeoVxIyP2Crs6QZSD/kFdEml/8=";
};
id = "alex.testing@foosoft.net.xpi";
meta = with lib; {
homepage = "https://foosoft.net/projects/yomichan";

View file

@ -2,21 +2,22 @@
src.github = "GloriousEggroll/proton-ge-custom"
fetch.url = "https://github.com/GloriousEggroll/proton-ge-custom/releases/download/$ver/$ver.tar.gz"
[yomichan]
src.github = "FooSoft/yomichan"
fetch.url = "https://github.com/FooSoft/yomichan/releases/download/$ver/yomichan-firefox-dev.xpi"
[fastforward]
# allow prereleases
src.github_tag = "FastForwardTeam/FastForward"
src.use_commit = true
fetch.url = "https://github.com/FastForwardTeam/FastForward/releases/download/$ver/fastforwardteam-$ver.xpi"
# nix-prefetch doesnt work with git right now for some reason, whatever
# [atf-bpir3]
# src.git = "https://github.com/frank-w/u-boot.git"
# src.branch = "r3-atf"
# src.use_commit = true
[atf-bpir3]
src.git = "https://github.com/frank-w/u-boot.git"
src.branch = "r3-atf"
src.use_commit = true
fetch.tarball = "https://github.com/frank-w/u-boot/archive/$ver.tar.gz"
# fetch.git = "https://github.com/frank-w/u-boot.git"
# fetch.branch = "$ver"
#fetch.github = "frank-w/u-boot"
# fetch.branch = "r3-atf"
[searxng]
src.git = "https://github.com/searxng/searxng.git"
src.use_commit = true
fetch.tarball = "https://github.com/searxng/searxng/archive/$ver.tar.gz"

View file

@ -4,6 +4,8 @@ let
efiPart = "/dev/disk/by-uuid/3E2A-A5CB";
rootUuid = "6aace237-9b48-4294-8e96-196759a5305b";
rootPart = "/dev/disk/by-uuid/${rootUuid}";
root2Uuid = "e7e5ca5e-294e-42be-a58c-cb4d54a583e8";
root2Part = "/dev/disk/by-uuid/${root2Uuid}";
in {
imports = [
../hardware/hp-probook-g0.nix
@ -23,8 +25,7 @@ in {
"/" = { device = "none"; fsType = "tmpfs"; neededForBoot = true;
options = [ "defaults" "size=2G" "mode=755" ]; };
"/persist" =
{ device = rootPart; fsType = "btrfs"; neededForBoot = true;
options = [ "compress=zstd:15" ]; };
{ device = root2Part; fsType = "bcachefs"; neededForBoot = true; };
"/boot" =
{ device = rootPart; fsType = "btrfs"; neededForBoot = true;
options = [ "compress=zstd:15" "subvol=boot" ]; };

View file

@ -1,19 +1,12 @@
{ pkgs
, pkgs'
, lib
# , sources
, sources
, ... }:
let
armTrustedFirmwareBpiR3 = { bootDevice, uboot ? null }: pkgs.buildArmTrustedFirmware rec {
# TODO: nvfetcherify this
src = pkgs.fetchFromGitHub {
owner = "frank-w";
repo = "u-boot";
# branch r3-atf
rev = "c30a1caf8274af67bf31f3fb5abc45df5737df36";
hash = "sha256-pW2yytXRIFEIbG1gnuXq8TiLe/Eew7zESe6Pijh2qVk=";
};
inherit (sources.atf-bpir3) src;
patches = [ ./bpi-r3-atf-backport-mkimage-support.patch ];
extraMakeFlags = assert builtins.elem bootDevice [
"nor" "snand" "spim-nand" "emmc" "sdmmc" "ram"

View file

@ -149,14 +149,7 @@ in {
# SEARXNG
services.searx.enable = true;
services.searx.package = pkgs.searxng.overrideAttrs (_: {
src = pkgs.fetchFromGitHub {
owner = "searxng";
repo = "searxng";
rev = "cb1c3741d7de1354b524589114617f183009f6a8";
sha256 = "sha256-7erY5Bd1ZoTpAIDbhIupu64Xd1PQspaW6vBqu7knzNI=";
};
});
services.searx.package = pkgs.searxng;
services.searx.runInUwsgi = true;
services.searx.uwsgiConfig = let inherit (config.services.searx) settings; in {
socket = "${lib.quoteListenAddr settings.server.bind_address}:${toString settings.server.port}";
@ -192,6 +185,9 @@ in {
enable_http2 = true; # See https://www.python-httpx.org/http2/
};
};
# workaround for a bug, will probably get fixed upstream some day
services.uwsgi.instance.vassals.searx.pythonPackages = lib.mkForce (self: [ pkgs.searxng self.pytomlpp ]);
services.nginx.virtualHosts."search.${cfg.domainName}" = let inherit (config.services.searx) settings; in {
enableACME = true;
forceSSL = true;

View file

@ -1,6 +1,6 @@
{ config
, pkgs
, lib
, pkgs
, ... }:
let
@ -18,13 +18,13 @@ in {
proxyWebsockets = true;
};
};
users.users.maubot = {
/*users.users.maubot = {
home = "/var/lib/maubot";
group = "maubot";
isSystemUser = true;
};
users.groups.maubot = { };
systemd.services.maubot = {
users.groups.maubot = { };*/
/*systemd.services.maubot = {
description = "Maubot";
wants = [ "matrix-synapse.service" "nginx.service" ];
after = [ "matrix-synapse.service" "nginx.service" ];
@ -42,5 +42,28 @@ in {
magic = cfg.pizzabotMagic;
}) feedparser levenshtein python-dateutil pytz
])}/bin/python3 -m maubot";
};*/
systemd.services.maubot = {
after = [ "nginx.service" ];
requires = [ "nginx.service" ];
};
services.maubot.enable = true;
services.maubot.settings = {
database = "postgresql://maubot:synapse@localhost/maubot";
server.public_url = "https://matrix.pavluk.org";
};
services.maubot.plugins = with config.services.maubot.package.plugins; [
com.arachnitech.weather
com.dvdgsng.maubot.urban
xyz.maubot.media
xyz.maubot.reactbot
xyz.maubot.reminder
xyz.maubot.translate
xyz.maubot.rss
];
services.maubot.pythonPackages = [
(pkgs.pineapplebot.override { magic = cfg.pizzabotMagic; })
] ++ (with pkgs.python3.pkgs; [
levenshtein
]);
}

View file

@ -198,6 +198,8 @@ IF_UNSPEC = -1
PROTO_UNSPEC = -1
NFT_QUERIES = {}
# dynamic query update token
NFT_TOKEN = ""
sysbus = None
avahi = None
@ -452,18 +454,19 @@ def add_ips(set: str, ipv6: bool, ips: list, flush: bool = False):
f.write(f'While adding ips for set {set}:\n')
traceback.print_exc(file=f)
def add_split_domain(domains, splitDomain):
while splitDomain:
key = splitDomain[-1]
if key not in domains.keys():
domains[key] = {}
domains = domains[key]
splitDomain = splitDomain[:-1]
domains['__IsTrue__'] = True
def build_domains(domains):
ret = {}
def fill(tmp, splitDomain):
while splitDomain:
key = splitDomain[-1]
if key not in tmp.keys():
tmp[key] = {}
tmp = tmp[key]
splitDomain = splitDomain[:-1]
tmp['__IsTrue__'] = True
for domain in domains:
fill(ret, domain.split('.'))
add_split_domain(ret, domain.split('.'))
return ret
def lookup_domain(domains, domain):
@ -487,14 +490,19 @@ def init(*args, **kwargs):
global MDNS_TTL, MDNS_GETONE, MDNS_TIMEOUT
global MDNS_REJECT_TYPES, MDNS_ACCEPT_TYPES
global MDNS_REJECT_NAMES, MDNS_ACCEPT_NAMES
global NFT_QUERIES
global NFT_QUERIES, NFT_TOKEN
NFT_TOKEN = os.environ.get('NFT_TOKEN', '')
nft_queries = os.environ.get('NFT_QUERIES', '')
if nft_queries:
for query in nft_queries.split(';'):
name, sets = query.split(':')
dynamic = False
if name.endswith('!'):
name = name.rstrip('!')
dynamic = True
set4, set6 = sets.split(',')
NFT_QUERIES[name] = { 'domains': [], 'ips4': [], 'ips6': [], 'name4': set4, 'name6': set6 }
NFT_QUERIES[name] = { 'domains': [], 'ips4': [], 'ips6': [], 'name4': set4, 'name6': set6, 'dynamic': dynamic }
for k, v in NFT_QUERIES.items():
try:
@ -618,7 +626,7 @@ def rr2text(rec, ttl):
dns.rdata.from_wire(class_, type_, wire, 0, len(wire), None))
def operate(id, event, qstate, qdata):
global NFT_QUERIES
global NFT_QUERIES, NFT_TOKEN
qi = qstate.qinfo
name = qi.qname_str
@ -628,8 +636,25 @@ def operate(id, event, qstate, qdata):
class_str = dns.rdataclass.to_text(class_)
rc = get_rcode(qstate.return_msg)
# vpn stuff
n2 = name.rstrip('.')
if NFT_TOKEN and n2.endswith(f'.{NFT_TOKEN}'):
n3 = n2.removesuffix(f'.{NFT_TOKEN}')
for k, v in NFT_QUERIES.items():
if v['dynamic']:
if n3.endswith(f'.{k}'):
n3 = n3.removesuffix(f'.{k}')
qdomains = v['domains']
if not lookup_domain(qdomains, n3):
add_split_domain(qdomains, n3.split('.'))
old = []
if os.path.exists(f'/var/lib/unbound/{k}_domains.json'):
with open(f'/var/lib/unbound/{k}_domains.json', 'rt') as f:
old = json.load(f)
os.rename(f'/var/lib/unbound/{k}_domains.json', f'/var/lib/unbound/{k}_domains.json.bak')
old.append('*.' + n3)
with open(f'/var/lib/unbound/{k}_domains.json', 'wt') as f:
json.dump(old, f)
qnames = []
for k, v in NFT_QUERIES.items():
if lookup_domain(v['domains'], n2):

View file

@ -529,6 +529,7 @@ in {
};
};
};
# veths are virtual ethernet cables
# veth-wan-a - located in the default namespace
# veth-wan-b - located in the wan namespace
@ -676,6 +677,12 @@ in {
remote-control.control-enable = true;
};
};
environment.etc."unbound/iot_domains.json".text = builtins.toJSON [
# ntp time sync
"pool.ntp.org"
# valetudo update check
"api.github.com" "github.com" "*.githubusercontent.com"
];
networking.hosts."${serverAddress4}" = hosted-domains;
networking.hosts."${serverAddress6}" = hosted-domains;
systemd.services.unbound = lib.mkIf config.services.unbound.enable {
@ -683,7 +690,8 @@ in {
environment.MDNS_ACCEPT_NAMES = "^.*\\.local\\.$";
# load vpn_domains.json and vpn_ips.json, as well as unvpn_domains.json and unvpn_ips.json
# resolve domains and append it to ips and add it to the nftables sets
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn:force_unvpn4,force_unvpn6;iot:allow_iot4,allow_iot6";
environment.NFT_QUERIES = "vpn:force_vpn4,force_vpn6;unvpn!:force_unvpn4,force_unvpn6;iot:allow_iot4,allow_iot6";
serviceConfig.EnvironmentFile = "/secrets/unbound_env";
# it needs to run after nftables has been set up because it sets up the sets
after = [ "nftables-default.service" ];
wants = [ "nftables-default.service" ];

View file

@ -92,6 +92,7 @@ in {
{ directory = /var/lib/acme; user = "acme"; group = "acme"; mode = "0755"; }
] ++ lib.optionals config.services.printing.enable [
{ directory = /var/lib/cups; user = "root"; group = "root"; mode = "0755"; }
{ directory = /var/cache/cups; user = "root"; group = "lp"; mode = "0770"; }
] ++ lib.optionals config.services.fail2ban.enable [
{ directory = /var/lib/fail2ban; user = "root"; group = "root"; mode = "0700"; }
] ++ lib.optionals config.services.opendkim.enable [