{ description = "A very basic flake"; inputs = { deploy-rs = { url = "github:serokell/deploy-rs"; inputs.nixpkgs.follows = "nixpkgs"; }; sd-images = { url = "git+https://git.asonix.dog/asonix/sd-images"; inputs.nixpkgs.follows = "nixpkgs"; }; nixpkgs.url = "github:asonix/nixpkgs/asonix/nixos-unstable-pinned"; sops-nix = { url = "github:Mic92/sops-nix"; inputs.nixpkgs.follows = "nixpkgs"; }; }; outputs = { self, deploy-rs, sd-images, nixpkgs, sops-nix }: let sharedModule = import ./modules/shared; btrbkModule = import ./modules/btrbk; dockerModule = import ./modules/docker; subvolumesModule = import ./modules/subvolumes; k3sModule = import ./modules/k3s; networkModule = import ./modules/network; makeConfig = { hostname, extraModules ? [ ] }: with sd-images.packages.x86_64-linux.modules; nixpkgs.lib.nixosSystem { system = "aarch64-linux"; modules = [ sops-nix.nixosModules.sops sharedModule customized baseModule { networking.hostName = hostname; } ] ++ extraModules; }; makeGenericK3sConfig = { hostname, enableK3s ? true, serverIp ? null, extraModules ? [ ] }: with sd-images.packages.x86_64-linux.modules; makeConfig { inherit hostname; extraModules = [ (if serverIp == null then k3sModule.server { enable = enableK3s; } else k3sModule.agent { inherit serverIp; enable = enableK3s; }) ({ config, pkgs, ... }: { services.lvm.enable = true; services.rpcbind.enable = true; environment.systemPackages = with pkgs; [ nfs-utils ]; }) ] ++ extraModules; }; makeBoardK3sConfig = module: { hostname, macAddress ? null, enableK3s ? true, selfIp, serverIp }: makeGenericK3sConfig { inherit hostname enableK3s serverIp; extraModules = [ (networkModule { inherit macAddress selfIp; }) module ]; }; makeRock64K3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.rock64; makeRockPro64K3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.rockPro64v2; makeQuartz64AK3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.quartz64a; makeSoQuartzK3sConfig = { hostname, enableK3s ? true, unlockMounts ? true, mountVolumes ? true, selfIp, serverIp ? null }: with sd-images.packages.x86_64-linux.modules; let device = "/dev/mapper/cryptdrive1"; device2 = "/dev/mapper/cryptdrive2"; subvolumes = [ "@k3s-config" ]; subvolumes2 = [ "@exports" ]; in makeGenericK3sConfig { inherit hostname serverIp; enableK3s = unlockMounts && mountVolumes && enableK3s; extraModules = [ soquartz-blade (networkModule { inherit selfIp; }) (btrbkModule { instances = [ { inherit subvolumes; mountDir = "/btrfs/nvme"; primaryIp = serverIp; } { subvolumes = subvolumes2; mountDir = "/btrfs/nvme2"; name = "nvme2"; } ]; }) (if unlockMounts && mountVolumes then (subvolumesModule { inherit device subvolumes; }) else { }) ({ config, pkgs, ... }: let keyFilePath = config.sops.secrets.k3sKeyFile.path; prepareNvme = '' #!/usr/bin/env bash set -e echo "Creating two partitions" fdisk -w always /dev/nvme0n1 << EOL g n 1 +50G n 2 w EOL echo "configure first part" echo "YES" | cryptsetup luksFormat /dev/nvme0n1p1 -d ${keyFilePath} cryptsetup luksOpen /dev/nvme0n1p1 cryptdrive1 -d ${keyFilePath} mkfs.btrfs /dev/mapper/cryptdrive1 mkdir -p /btrfs/nvme mount /dev/mapper/cryptdrive1 /btrfs/nvme btrfs subvolume create /btrfs/nvme/@k3s-config btrfs subvolume create /btrfs/nvme/@snapshots btrfs subvolume create /btrfs/nvme/@swap btrfs subvolume create /btrfs/nvme/@var-lib-rancher btrfs subvolume create /btrfs/nvme/@var-log-pods btrfs filesystem mkswapfile -s 4g /btrfs/nvme/@swap/file umount /btrfs/nvme cryptsetup luksClose cryptdrive1 echo "configure second part" echo "YES" | cryptsetup luksFormat /dev/nvme0n1p2 -d ${keyFilePath} cryptsetup luksOpen /dev/nvme0n1p2 cryptdrive2 -d ${keyFilePath} mkfs.btrfs /dev/mapper/cryptdrive2 mkdir -p /btrfs/nvme2 mount /dev/mapper/cryptdrive2 /btrfs/nvme2 btrfs subvolume create /btrfs/nvme2/@exports btrfs subvolume create /btrfs/nvme2/@snapshots umount /btrfs/nvme2 cryptsetup luksClose cryptdrive2 ''; in { sops.secrets.k3sKeyFile = { format = "binary"; sopsFile = ./secrets/k3sKeyFile.bin; }; environment.systemPackages = with pkgs; [ (writeShellScriptBin "prepare-nvme" prepareNvme) ]; environment.etc.crypttab = { enable = unlockMounts; text = '' cryptdrive1 /dev/nvme0n1p1 ${keyFilePath} luks cryptdrive2 /dev/nvme0n1p2 ${keyFilePath} luks ''; }; fileSystems = let defaultOptions = [ "defaults" "rw" "compress=zstd" ]; fileSystemConfig = { "/btrfs/nvme" = { inherit device; fsType = "btrfs"; options = defaultOptions; }; "/btrfs/nvme2" = { device = device2; fsType = "btrfs"; options = defaultOptions; }; "/exports" = { device = device2; fsType = "btrfs"; options = defaultOptions ++ [ "subvol=@exports" ]; }; "/swap" = { inherit device; fsType = "btrfs"; options = defaultOptions ++ [ "subvol=@swap" ]; }; "/var/lib/rancher" = { inherit device; fsType = "btrfs"; options = defaultOptions ++ [ "subvol=@var-lib-rancher" ]; }; "/var/log/pods" = { inherit device; fsType = "btrfs"; options = defaultOptions ++ [ "subvol=@var-log-pods" ]; }; }; in if unlockMounts && mountVolumes then fileSystemConfig else { }; swapDevices = let swapFile = { device = "/swap/file"; size = (1024 * 4); }; in if unlockMounts && mountVolumes then [ swapFile ] else [ ]; services.nfs.server = { enable = unlockMounts && mountVolumes; exports = '' /exports 192.168.20.0/24(rw,sync,fsid=0,no_subtree_check) /exports/k3s 192.168.20.0/24(rw,nohide,insecure,sync,no_subtree_check,no_root_squash) ''; }; }) ]; }; makeBuildConfig = { hostname, macAddress ? null, selfIp }: with sd-images.packages.x86_64-linux.modules; let device = "/btrfs.4G"; mountDir = "/btrfs/loop"; subvolumes = [ "@build-cfg" ]; in makeConfig { inherit hostname; extraModules = [ rockPro64v2 dockerModule (networkModule { inherit macAddress selfIp; }) (btrbkModule { instances = [ { inherit mountDir subvolumes; } ]; }) (subvolumesModule { inherit device subvolumes; }) { fileSystems."${mountDir}" = { inherit device; fsType = "btrfs"; options = [ "defaults" "compress=zstd" "rw" "loop" ]; }; } ]; }; makeGarageConfig = { hostname, selfIp, primaryIp ? null, unlockMounts ? true, mountVolumes ? true }: with sd-images.packages.x86_64-linux.modules; let device = "/dev/mapper/cryptdrive1"; mountDir = "/btrfs/hdd"; subvolumes = [ "@gitea" "@gitea-conf" "@pihole" "@docker-cfg" ]; garageVolumes = [ "@garage-cluster" "@garage-cluster-config" ]; in makeConfig { inherit hostname; extraModules = [ rockPro64v2 dockerModule (networkModule { inherit selfIp; }) (btrbkModule { instances = [ { inherit mountDir primaryIp subvolumes; } { inherit mountDir; name = "garage"; subvolumes = garageVolumes; } ]; }) (if primaryIp == null && unlockMounts && mountVolumes then (subvolumesModule { inherit device subvolumes; }) else { }) (if unlockMounts && mountVolumes then (subvolumesModule { inherit device; subvolumes = garageVolumes; }) else { }) ({ config, pkgs, ... }: let keyFile = "nextcloudKeyFile"; keyFilePath = config.sops.secrets."${keyFile}".path; prepareDrives = '' fdisk -w always /dev/sda << EOL g n 1 w EOL fdisk -w always /dev/sdb << EOL g n 1 w EOL fdisk -w always /dev/sdc << EOL g n 1 w EOL echo "YES" | cryptsetup luksFormat /dev/sda1 -d ${keyFilePath} cryptsetup luksOpen /dev/sda1 cryptdrive1 -d ${keyFilePath} echo "YES" | cryptsetup luksFormat /dev/sdb1 -d ${keyFilePath} cryptsetup luksOpen /dev/sdb1 cryptdrive2 -d ${keyFilePath} echo "YES" | cryptsetup luksFormat /dev/sdc1 -d ${keyFilePath} cryptsetup luksOpen /dev/sdc1 cryptdrive3 -d ${keyFilePath} mkfs.btrfs -d raid1c3 -m raid1c3 /dev/mapper/cryptdrive1 /dev/mapper/cryptdrive2 /dev/mapper/cryptdrive3 mkdir -p /btrfs/hdd mount /dev/mapper/cryptdrive1 /btrfs/hdd btrfs subvolume create /btrfs/hdd/@garage-cluster btrfs subvolume create /btrfs/hdd/@garage-cluster-config btrfs subvolume create /btrfs/hdd/@snapshots umount /btrfs/hdd cryptsetup luksClose cryptdrive1 cryptsetup luksClose cryptdrive2 cryptsetup luksClose cryptdrive3 ''; in { sops.secrets.nextcloudKeyFile = { format = "binary"; sopsFile = ./secrets/nextcloudKeyFile.bin; }; environment.systemPackages = with pkgs; [ (writeShellScriptBin "prepare-drives" prepareDrives) minio-client ]; environment.etc.crypttab = { enable = unlockMounts; text = '' cryptdrive1 /dev/sda1 ${keyFilePath} luks cryptdrive2 /dev/sdb1 ${keyFilePath} luks cryptdrive3 /dev/sdc1 ${keyFilePath} luks ''; }; fileSystems = let fileSystemConfig = { "${mountDir}" = { inherit device; fsType = "btrfs"; options = [ "defaults" "compress=zstd" "rw" ]; }; }; in if unlockMounts && mountVolumes then fileSystemConfig else { }; networking.firewall.enable = false; }) ]; }; makePostgresConfig = { hostname, selfIp, macAddress, keyFile, primaryIp ? null }: with sd-images.packages.x86_64-linux.modules; let device = "/dev/mapper/cryptdrive1"; mountDir = "/btrfs/ssd"; subvolumes = [ "@postgres" "@postgres-cfg" ]; in makeConfig { inherit hostname; extraModules = [ rock64 dockerModule (networkModule { inherit macAddress selfIp; }) (btrbkModule { instances = [{ inherit mountDir primaryIp subvolumes; }]; }) (if primaryIp == null then (subvolumesModule { inherit device subvolumes; }) else { }) ({ config, ... }: let keyFilePath = config.sops.secrets."${keyFile}".path; in { sops.secrets.${keyFile} = { format = "binary"; sopsFile = ./secrets/${keyFile}.bin; }; environment.etc.crypttab = { enable = true; text = '' cryptdrive1 /dev/sda1 ${keyFilePath} luks ''; }; fileSystems."${mountDir}" = { inherit device; fsType = "btrfs"; options = [ "defaults" "compress=zstd" "rw" ]; }; }) ]; }; in { nixosConfigurations = { garage1 = makeGarageConfig { hostname = "garage1"; selfIp = "192.168.20.21"; # primaryIp = "192.168.20.28"; }; garage2 = makeGarageConfig { hostname = "garage2"; selfIp = "192.168.20.28"; primaryIp = "192.168.20.21"; }; garage3 = makeGarageConfig { hostname = "garage3"; selfIp = "192.168.20.29"; primaryIp = "192.168.20.21"; }; redtail1 = makePostgresConfig { hostname = "redtail1"; selfIp = "192.168.20.23"; macAddress = "02:fe:30:d8:cf:64"; keyFile = "redtailKeyFile"; primaryIp = "192.168.20.24"; }; redtail2 = makePostgresConfig { hostname = "redtail2"; selfIp = "192.168.20.24"; macAddress = "02:8a:70:2a:a8:5e"; keyFile = "redtailKeyFile"; # primaryIp = "192.168.20.23"; }; whitestorm1 = makePostgresConfig { hostname = "whitestorm1"; selfIp = "192.168.20.26"; macAddress = "02:71:18:30:fc:0f"; keyFile = "whitestormKeyFile"; # primaryIp = "192.168.20.27"; }; whitestorm2 = makePostgresConfig { hostname = "whitestorm2"; selfIp = "192.168.20.27"; macAddress = "02:ff:ce:a9:d3:74"; keyFile = "whitestormKeyFile"; primaryIp = "192.168.20.11"; }; build2 = makeBuildConfig { hostname = "build2"; selfIp = "192.168.20.101"; }; k3s1 = makeSoQuartzK3sConfig { hostname = "k3s1"; selfIp = "192.168.20.120"; }; k3s2 = makeSoQuartzK3sConfig { hostname = "k3s2"; selfIp = "192.168.20.121"; serverIp = "192.168.20.120"; }; k3s3 = makeSoQuartzK3sConfig { hostname = "k3s3"; selfIp = "192.168.20.122"; serverIp = "192.168.20.120"; }; k3s4 = makeSoQuartzK3sConfig { hostname = "k3s4"; selfIp = "192.168.20.123"; serverIp = "192.168.20.120"; }; k3s5 = makeSoQuartzK3sConfig { hostname = "k3s5"; selfIp = "192.168.20.124"; serverIp = "192.168.20.120"; }; k3s6 = makeSoQuartzK3sConfig { hostname = "k3s6"; selfIp = "192.168.20.125"; serverIp = "192.168.20.120"; }; k3s-rock1 = makeRock64K3sConfig { hostname = "k3s-rock1"; selfIp = "192.168.20.20"; serverIp = "192.168.20.120"; macAddress = "02:36:51:19:8c:5d"; }; k3s-rock2 = makeRock64K3sConfig { hostname = "k3s-rock2"; selfIp = "192.168.20.111"; serverIp = "192.168.20.120"; macAddress = "02:af:fa:0e:c5:41"; }; k3s-rock3 = makeRock64K3sConfig { hostname = "k3s-rock3"; selfIp = "192.168.20.112"; serverIp = "192.168.20.120"; macAddress = "02:86:c1:62:77:51"; }; k3s-rock4 = makeRock64K3sConfig { hostname = "k3s-rock4"; selfIp = "192.168.20.113"; serverIp = "192.168.20.120"; macAddress = "02:7a:2e:2c:bb:99"; }; k3s-rock5 = makeRock64K3sConfig { hostname = "k3s-rock5"; selfIp = "192.168.20.114"; serverIp = "192.168.20.120"; macAddress = "02:76:67:0f:a0:b7"; }; k3s-rock6 = makeRock64K3sConfig { hostname = "k3s-rock6"; selfIp = "192.168.20.115"; serverIp = "192.168.20.120"; macAddress = "02:6b:c5:f0:d7:6d"; }; k3s-quartza1 = makeQuartz64AK3sConfig { hostname = "k3s-quartza1"; selfIp = "192.168.20.160"; serverIp = "192.168.20.120"; }; k3s-rockpro1 = makeRockPro64K3sConfig { hostname = "k3s-rockpro1"; selfIp = "192.168.20.180"; serverIp = "192.168.20.120"; }; }; deploy.nodes = let nodes = [ { name = "redtail1"; ip = "192.168.20.23"; } { name = "whitestorm2"; ip = "192.168.20.27"; } { name = "garage1"; ip = "192.168.20.21"; } { name = "garage2"; ip = "192.168.20.28"; } { name = "garage3"; ip = "192.168.20.29"; } { name = "build2"; ip = "192.168.20.101"; } { name = "k3s1"; ip = "192.168.20.120"; } { name = "k3s2"; ip = "192.168.20.121"; } { name = "k3s3"; ip = "192.168.20.122"; } { name = "k3s4"; ip = "192.168.20.123"; } { name = "k3s5"; ip = "192.168.20.124"; } { name = "k3s6"; ip = "192.168.20.125"; } { name = "k3s-rock1"; ip = "192.168.20.20"; } { name = "k3s-rock2"; ip = "192.168.20.111"; } { name = "k3s-rock3"; ip = "192.168.20.112"; } { name = "k3s-rock4"; ip = "192.168.20.113"; } { name = "k3s-rock5"; ip = "192.168.20.114"; } { name = "k3s-rock6"; ip = "192.168.20.115"; } { name = "k3s-quartza1"; ip = "192.168.20.160"; } { name = "k3s-rockpro1"; ip = "192.168.20.180"; } ]; in builtins.foldl' (acc: { name, ip }: acc // { ${name} = { hostname = ip; profiles.system = { sshUser = "asonix"; user = "root"; magicRollback = false; sshOpts = [ "-i" "/home/asonix/.ssh/kube-rsa" "-t" ]; path = deploy-rs.lib.aarch64-linux.activate.nixos self.nixosConfigurations.${name}; }; }; }) { } nodes; }; }