dotfiles/system/common/vfio.nix

142 lines
4.8 KiB
Nix
Raw Normal View History

2023-01-24 02:24:40 +07:00
{ config, lib, pkgs, ... }:
{
options.vfio = with lib; mkOption {
type = types.submodule {
options = {
enable = mkOption {
type = types.bool;
default = false;
description = "Enable AMD GPU passthrough config (no intel/nvidia support since I can't test it)";
};
libvirtdGroup = mkOption {
type = with types; listOf str;
default = ["user"];
description = "Users to add to libvirtd group";
};
intelCpu = mkOption {
type = types.bool;
default = false;
description = "Whether the CPU is Intel (untested)";
};
passGpuAtBoot = mkOption {
type = types.bool;
default = false;
description = "Whether to pass the GPU at boot (can be more stable)";
};
pciIDs = mkOption {
type = with types; listOf str;
default = [];
description = "PCI passthrough IDs";
};
lookingGlass = mkOption {
type = types.submodule {
options = {
enable = mkOption {
type = types.bool;
default = true;
description = "Enable Looking Glass integration";
};
ivshmem = mkOption {
type = with types; listOf (submodule {
options = {
size = mkOption {
type = types.int;
default = 32;
description = "IVSHMEM size in MB: https://looking-glass.io/docs/B6/install/#determining-memory";
};
owner = mkOption {
type = types.str;
description = "IVSHMEM device owner";
};
};
});
default = [];
example = [{ size = 32; owner = "user"; }];
description = "IVSHMEM/kvmfr config (multiple devices can be created: /dev/kvmfr0, /dev/kvmfr1, and so on)";
};
};
};
default = {};
description = "Looking glass config";
};
};
};
default = {};
description = "VFIO settings";
};
config = lib.mkIf config.vfio.enable
(let
cfg = config.vfio;
gpuIDs = lib.concatStringsSep "," cfg.pciIDs;
enableIvshmem = config.vfio.lookingGlass.enable && (builtins.length config.vfio.lookingGlass.ivshmem) > 0;
in {
boot = {
initrd.kernelModules = [
(if cfg.intelCpu then "kvm-intel" else "kvm-amd")
] ++ (if cfg.passGpuAtBoot then [
"vfio"
"vfio_iommu_type1"
"vfio_pci"
"vfio_virqfd"
] else []);
extraModulePackages =
with config.boot.kernelPackages;
lib.mkIf enableIvshmem [ kvmfr ];
extraModprobeConfig = let ivshmemConfig = if enableIvshmem then ''
options kvmfr static_size_mb=${lib.concatStringsSep "," (map (x: toString x.size) cfg.lookingGlass.ivshmem)}
'' else ""; in ''
options vfio-pci ids=${gpuIDs}
options kvm ignore_msrs=1
${ivshmemConfig}
'';
kernelParams = [
(if cfg.intelCpu then "intel_iommu=on" else "amd_iommu=on")
"iommu=pt"
];
kernelModules = [
"vhost-net"
] ++ (if cfg.passGpuAtBoot then [] else [ "vfio_virqfd" ])
++ (if enableIvshmem then [ "kvmfr" ] else []);
};
services.udev.extraRules = lib.mkIf enableIvshmem
(lib.concatStringsSep
"\n"
(lib.imap0
(i: ivshmem: ''
SUBSYSTEM=="kvmfr", KERNEL=="kvmfr${toString i}", OWNER="${ivshmem.owner}", GROUP="kvm", MODE="0660"
'')
cfg.lookingGlass.ivshmem));
# disable early KMS so GPU can be properly unbound
hardware.amdgpu.loadInInitrd = false;
hardware.opengl.enable = true;
# needed for virt-manager
programs.dconf.enable = true;
virtualisation.libvirtd = {
enable = true;
onBoot = "ignore";
onShutdown = "shutdown";
qemu = {
ovmf.enable = true;
# Full is needed for TPM and secure boot emulation
ovmf.packages = [ pkgs.OVMFFull ];
# TPM emulation
swtpm.enable = true;
verbatimConfig = ''
nvram = [ "${pkgs.OVMFFull}/FV/OVMF.fd:${pkgs.OVMFFull}/FV/OVMF_VARS.fd" ]
cgroup_device_acl = [
"/dev/kvmfr0",
"/dev/vfio/vfio", "/dev/vfio/11", "/dev/vfio/12",
"/dev/null", "/dev/full", "/dev/zero",
"/dev/random", "/dev/urandom",
"/dev/ptmx", "/dev/kvm"
]
'';
# might disable this later
runAsRoot = true;
};
};
virtualisation.spiceUSBRedirection.enable = true;
users.groups.libvirtd.members = [ "root" ] ++ cfg.libvirtdGroup;
});
}