sbc-deploys/flake.nix

{
  description = "A very basic flake";

  inputs = {
    deploy-rs = {
      url = "github:serokell/deploy-rs";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    sd-images = {
      url = "git+https://git.asonix.dog/asonix/sd-images";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    nixpkgs.url = "github:asonix/nixpkgs/asonix/nixos-unstable-pinned";
    sops-nix = {
      url = "github:Mic92/sops-nix";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs = { self, deploy-rs, sd-images, nixpkgs, sops-nix }:
    let
      sharedModule = import ./modules/shared;
      btrbkModule = import ./modules/btrbk;
      dockerModule = import ./modules/docker;
      subvolumesModule = import ./modules/subvolumes;
      k3sModule = import ./modules/k3s;
      networkModule = import ./modules/network;

      makeConfig = { hostname, extraModules ? [ ] }:
        with sd-images.packages.x86_64-linux.modules;
        nixpkgs.lib.nixosSystem {
          system = "aarch64-linux";
          modules = [
            sops-nix.nixosModules.sops
            sharedModule
            customized
            baseModule
            {
              networking.hostName = hostname;
            }
          ] ++ extraModules;
        };

      makeGenericK3sConfig = { hostname, enableK3s ? true, serverIp ? null, extraModules ? [ ] }:
        with sd-images.packages.x86_64-linux.modules;
        makeConfig {
          inherit hostname;

          extraModules = [
            (if serverIp == null
            then
              k3sModule.server { enable = enableK3s; }
            else
              k3sModule.agent {
                inherit serverIp;
                enable = enableK3s;
              })
            ({ config, pkgs, ... }:
              {
                services.lvm.enable = true;
                services.rpcbind.enable = true;

                environment.systemPackages = with pkgs; [
                  nfs-utils
                ];
              })
          ] ++ extraModules;
        };

      makeBoardK3sConfig = module: { hostname, macAddress ? null, enableK3s ? true, selfIp, serverIp }:
        makeGenericK3sConfig {
          inherit hostname enableK3s serverIp;

          extraModules = [
            (networkModule { inherit macAddress selfIp; })
            module
          ];
        };

      makeRock64K3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.rock64;

      makeRockPro64K3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.rockPro64v2;

      makeQuartz64AK3sConfig = makeBoardK3sConfig sd-images.packages.x86_64-linux.modules.quartz64a;

      makeSoQuartzK3sConfig = { hostname, enableK3s ? true, unlockMounts ? true, mountVolumes ? true, selfIp, serverIp ? null }:
        with sd-images.packages.x86_64-linux.modules;
        let
          device = "/dev/mapper/cryptdrive1";
          device2 = "/dev/mapper/cryptdrive2";
          subvolumes = [
            "@k3s-config"
          ];
          subvolumes2 = [
            "@exports"
          ];
        in
        makeGenericK3sConfig {
          inherit hostname serverIp;

          enableK3s = unlockMounts && mountVolumes && enableK3s;

          extraModules = [
            soquartz-blade
            (networkModule {
              inherit selfIp;
            })
            (btrbkModule {
              instances = [
                {
                  inherit subvolumes;
                  mountDir = "/btrfs/nvme";
                  primaryIp = serverIp;
                }
                {
                  subvolumes = subvolumes2;
                  mountDir = "/btrfs/nvme2";
                  name = "nvme2";
                }
              ];
            })
            (if unlockMounts && mountVolumes then
              (subvolumesModule {
                inherit device subvolumes;
              }) else { })
            ({ config, pkgs, ... }:
              let
                keyFilePath = config.sops.secrets.k3sKeyFile.path;
                prepareNvme = ''
                  #!/usr/bin/env bash

                  set -e

                  echo "Creating two partitions"
                  fdisk -w always /dev/nvme0n1 << EOL
                  g
                  n
                  1

                  +50G
                  n
                  2


                  w
                  EOL

                  echo "configure first part"

                  echo "YES" | cryptsetup luksFormat /dev/nvme0n1p1 -d ${keyFilePath}
                  cryptsetup luksOpen /dev/nvme0n1p1 cryptdrive1 -d ${keyFilePath}

                  mkfs.btrfs /dev/mapper/cryptdrive1

                  mkdir -p /btrfs/nvme
                  mount /dev/mapper/cryptdrive1 /btrfs/nvme

                  btrfs subvolume create /btrfs/nvme/@k3s-config
                  btrfs subvolume create /btrfs/nvme/@snapshots
                  btrfs subvolume create /btrfs/nvme/@swap
                  btrfs subvolume create /btrfs/nvme/@var-lib-rancher
                  btrfs subvolume create /btrfs/nvme/@var-log-pods

                  btrfs filesystem mkswapfile -s 4g /btrfs/nvme/@swap/file

                  umount /btrfs/nvme

                  cryptsetup luksClose cryptdrive1

                  echo "configure second part"

                  echo "YES" | cryptsetup luksFormat /dev/nvme0n1p2 -d ${keyFilePath}
                  cryptsetup luksOpen /dev/nvme0n1p2 cryptdrive2 -d ${keyFilePath}

                  mkfs.btrfs /dev/mapper/cryptdrive2

                  mkdir -p /btrfs/nvme2
                  mount /dev/mapper/cryptdrive2 /btrfs/nvme2

                  btrfs subvolume create /btrfs/nvme2/@exports
                  btrfs subvolume create /btrfs/nvme2/@snapshots

                  umount /btrfs/nvme2

                  cryptsetup luksClose cryptdrive2
                '';
              in
              {
                sops.secrets.k3sKeyFile = {
                  format = "binary";
                  sopsFile = ./secrets/k3sKeyFile.bin;
                };

                environment.systemPackages = with pkgs; [
                  (writeShellScriptBin "prepare-nvme" prepareNvme)
                ];

                environment.etc.crypttab = {
                  enable = unlockMounts;
                  text = ''
                    cryptdrive1 /dev/nvme0n1p1 ${keyFilePath} luks
                    cryptdrive2 /dev/nvme0n1p2 ${keyFilePath} luks
                  '';
                };

                fileSystems =
                  let
                    defaultOptions = [ "defaults" "rw" "compress=zstd" ];
                    fileSystemConfig =
                      {
                        "/btrfs/nvme" = {
                          inherit device;
                          fsType = "btrfs";
                          options = defaultOptions;
                        };
                        "/btrfs/nvme2" = {
                          device = device2;
                          fsType = "btrfs";
                          options = defaultOptions;
                        };
                        "/exports" = {
                          device = device2;
                          fsType = "btrfs";
                          options = defaultOptions ++ [ "subvol=@exports" ];
                        };
                        "/swap" = {
                          inherit device;
                          fsType = "btrfs";
                          options = defaultOptions ++ [ "subvol=@swap" ];
                        };
                        "/var/lib/rancher" = {
                          inherit device;
                          fsType = "btrfs";
                          options = defaultOptions ++ [ "subvol=@var-lib-rancher" ];
                        };
                        "/var/log/pods" = {
                          inherit device;
                          fsType = "btrfs";
                          options = defaultOptions ++ [ "subvol=@var-log-pods" ];
                        };
                      };
                  in
                  if unlockMounts && mountVolumes then
                    fileSystemConfig
                  else
                    { };

                swapDevices =
                  let
                    swapFile = {
                      device = "/swap/file";
                      size = (1024 * 4);
                    };
                  in
                  if unlockMounts && mountVolumes then
                    [ swapFile ]
                  else
                    [ ];

                services.nfs.server = {
                  enable = unlockMounts && mountVolumes;
                  exports = ''
                    /exports  192.168.20.0/24(rw,sync,fsid=0,no_subtree_check)
                    /exports/k3s  192.168.20.0/24(rw,nohide,insecure,sync,no_subtree_check,no_root_squash)
                  '';
                };
              })
          ];
        };

      makeBuildConfig = { hostname, macAddress ? null, selfIp }:
        with sd-images.packages.x86_64-linux.modules;
        let
          device = "/btrfs.4G";
          mountDir = "/btrfs/loop";
          subvolumes = [
            "@build-cfg"
          ];
        in
        makeConfig {
          inherit hostname;

          extraModules = [
            rockPro64v2
            dockerModule
            (networkModule { inherit macAddress selfIp; })
            (btrbkModule {
              instances = [
                { inherit mountDir subvolumes; }
              ];
            })
            (subvolumesModule {
              inherit device subvolumes;
            })
            {
              fileSystems."${mountDir}" = {
                inherit device;
                fsType = "btrfs";
                options = [ "defaults" "compress=zstd" "rw" "loop" ];
              };
            }
          ];
        };

      makeGarageConfig = { hostname, selfIp, primaryIp ? null, unlockMounts ? true, mountVolumes ? true }:
        with sd-images.packages.x86_64-linux.modules;
        let
          device = "/dev/mapper/cryptdrive1";
          mountDir = "/btrfs/hdd";
          subvolumes = [
            "@gitea"
            "@gitea-conf"
            "@pihole"
            "@docker-cfg"
          ];
          garageVolumes = [
            "@garage-cluster"
            "@garage-cluster-config"
          ];
        in
        makeConfig {
          inherit hostname;

          extraModules = [
            rockPro64v2
            dockerModule
            (networkModule { inherit selfIp; })
            (btrbkModule {
              instances = [
                {
                  inherit mountDir primaryIp subvolumes;
                }
                {
                  inherit mountDir;
                  name = "garage";
                  subvolumes = garageVolumes;
                }
              ];
            })
            (if primaryIp == null && unlockMounts && mountVolumes then
              (subvolumesModule {
                inherit device subvolumes;
              }) else { })
            (if unlockMounts && mountVolumes then
              (subvolumesModule {
                inherit device;
                subvolumes = garageVolumes;
              }) else { })
            ({ config, pkgs, ... }:
              let
                keyFile = "nextcloudKeyFile";
                keyFilePath = config.sops.secrets."${keyFile}".path;
                prepareDrives = ''
                  fdisk -w always /dev/sda << EOL
                  g
                  n
                  1


                  w
                  EOL

                  fdisk -w always /dev/sdb << EOL
                  g
                  n
                  1


                  w
                  EOL

                  fdisk -w always /dev/sdc << EOL
                  g
                  n
                  1


                  w
                  EOL

                  echo "YES" | cryptsetup luksFormat /dev/sda1 -d ${keyFilePath}
                  cryptsetup luksOpen /dev/sda1 cryptdrive1 -d ${keyFilePath}

                  echo "YES" | cryptsetup luksFormat /dev/sdb1 -d ${keyFilePath}
                  cryptsetup luksOpen /dev/sdb1 cryptdrive2 -d ${keyFilePath}

                  echo "YES" | cryptsetup luksFormat /dev/sdc1 -d ${keyFilePath}
                  cryptsetup luksOpen /dev/sdc1 cryptdrive3 -d ${keyFilePath}

                  mkfs.btrfs -d raid1c3 -m raid1c3 /dev/mapper/cryptdrive1 /dev/mapper/cryptdrive2 /dev/mapper/cryptdrive3

                  mkdir -p /btrfs/hdd
                  mount /dev/mapper/cryptdrive1 /btrfs/hdd

                  btrfs subvolume create /btrfs/hdd/@garage-cluster
                  btrfs subvolume create /btrfs/hdd/@garage-cluster-config
                  btrfs subvolume create /btrfs/hdd/@snapshots

                  umount /btrfs/hdd

                  cryptsetup luksClose cryptdrive1
                  cryptsetup luksClose cryptdrive2
                  cryptsetup luksClose cryptdrive3
                '';
              in
              {
                sops.secrets.nextcloudKeyFile = {
                  format = "binary";
                  sopsFile = ./secrets/nextcloudKeyFile.bin;
                };

                environment.systemPackages = with pkgs; [
                  (writeShellScriptBin "prepare-drives" prepareDrives)
                  minio-client
                ];

                environment.etc.crypttab = {
                  enable = unlockMounts;
                  text = ''
                    cryptdrive1 /dev/sda1 ${keyFilePath} luks
                    cryptdrive2 /dev/sdb1 ${keyFilePath} luks
                    cryptdrive3 /dev/sdc1 ${keyFilePath} luks
                  '';
                };

                fileSystems =
                  let fileSystemConfig = {
                    "${mountDir}" = {
                      inherit device;
                      fsType = "btrfs";
                      options = [ "defaults" "compress=zstd" "rw" ];
                    };
                  };
                  in
                  if unlockMounts && mountVolumes then fileSystemConfig else { };

                networking.firewall.enable = false;
              })
          ];
        };

      makePostgresConfig = { hostname, selfIp, macAddress, keyFile, primaryIp ? null }:
        with sd-images.packages.x86_64-linux.modules;
        let
          device = "/dev/mapper/cryptdrive1";
          mountDir = "/btrfs/ssd";
          subvolumes = [
            "@postgres"
            "@postgres-cfg"
          ];
        in
        makeConfig {
          inherit hostname;

          extraModules = [
            rock64
            dockerModule
            (networkModule { inherit macAddress selfIp; })
            (btrbkModule {
              instances = [{
                inherit mountDir primaryIp subvolumes;
              }];
            })
            (if primaryIp == null then
              (subvolumesModule {
                inherit device subvolumes;
              }) else { })
            ({ config, ... }:
              let
                keyFilePath = config.sops.secrets."${keyFile}".path;
              in
              {
                sops.secrets.${keyFile} = {
                  format = "binary";
                  sopsFile = ./secrets/${keyFile}.bin;
                };

                environment.etc.crypttab = {
                  enable = true;
                  text = ''
                    cryptdrive1 /dev/sda1 ${keyFilePath} luks
                  '';
                };

                fileSystems."${mountDir}" = {
                  inherit device;
                  fsType = "btrfs";
                  options = [ "defaults" "compress=zstd" "rw" ];
                };
              })
          ];
        };

      deployer = { hostname, configuration }: {
        hostname = hostname;
        profiles.system = {
          sshUser = "asonix";
          user = "root";
          magicRollback = false;
          sshOpts = [
            "-i"
            "/home/asonix/.ssh/kube-rsa"
            "-t"
          ];
          path = deploy-rs.lib.aarch64-linux.activate.nixos configuration;
        };
      };
    in
    {
      nixosConfigurations = {
        garage1 = makeGarageConfig {
          hostname = "garage1";
          selfIp = "192.168.20.21";
          # primaryIp = "192.168.20.28";
        };

        garage2 = makeGarageConfig {
          hostname = "garage2";
          selfIp = "192.168.20.28";
          primaryIp = "192.168.20.21";
        };

        garage3 = makeGarageConfig {
          hostname = "garage3";
          selfIp = "192.168.20.29";
          primaryIp = "192.168.20.21";
        };

        redtail1 = makePostgresConfig {
          hostname = "redtail1";
          selfIp = "192.168.20.23";
          macAddress = "02:fe:30:d8:cf:64";
          keyFile = "redtailKeyFile";
          primaryIp = "192.168.20.24";
        };

        redtail2 = makePostgresConfig {
          hostname = "redtail2";
          selfIp = "192.168.20.24";
          macAddress = "02:8a:70:2a:a8:5e";
          keyFile = "redtailKeyFile";
          # primaryIp = "192.168.20.23";
        };

        whitestorm1 = makePostgresConfig {
          hostname = "whitestorm1";
          selfIp = "192.168.20.26";
          macAddress = "02:71:18:30:fc:0f";
          keyFile = "whitestormKeyFile";
          # primaryIp = "192.168.20.27";
        };

        whitestorm2 = makePostgresConfig {
          hostname = "whitestorm2";
          selfIp = "192.168.20.27";
          macAddress = "02:ff:ce:a9:d3:74";
          keyFile = "whitestormKeyFile";
          primaryIp = "192.168.20.11";
        };

        build2 = makeBuildConfig {
          hostname = "build2";
          selfIp = "192.168.20.101";
        };

        k3s1 = makeSoQuartzK3sConfig {
          hostname = "k3s1";
          selfIp = "192.168.20.120";
        };

        k3s2 = makeSoQuartzK3sConfig {
          hostname = "k3s2";
          selfIp = "192.168.20.121";
          serverIp = "192.168.20.120";
        };

        k3s3 = makeSoQuartzK3sConfig {
          hostname = "k3s3";
          selfIp = "192.168.20.122";
          serverIp = "192.168.20.120";
        };

        k3s4 = makeSoQuartzK3sConfig {
          hostname = "k3s4";
          selfIp = "192.168.20.123";
          serverIp = "192.168.20.120";
        };

        k3s5 = makeSoQuartzK3sConfig {
          hostname = "k3s5";
          selfIp = "192.168.20.124";
          serverIp = "192.168.20.120";
        };

        k3s6 = makeSoQuartzK3sConfig {
          hostname = "k3s6";
          selfIp = "192.168.20.125";
          serverIp = "192.168.20.120";
        };

        k3s-rock1 = makeRock64K3sConfig {
          hostname = "k3s-rock1";
          selfIp = "192.168.20.20";
          serverIp = "192.168.20.120";
          macAddress = "02:36:51:19:8c:5d";
        };

        k3s-rock2 = makeRock64K3sConfig {
          hostname = "k3s-rock2";
          selfIp = "192.168.20.111";
          serverIp = "192.168.20.120";
          macAddress = "02:af:fa:0e:c5:41";
        };

        k3s-rock3 = makeRock64K3sConfig {
          hostname = "k3s-rock3";
          selfIp = "192.168.20.112";
          serverIp = "192.168.20.120";
          macAddress = "02:86:c1:62:77:51";
        };

        k3s-rock4 = makeRock64K3sConfig {
          hostname = "k3s-rock4";
          selfIp = "192.168.20.113";
          serverIp = "192.168.20.120";
          macAddress = "02:7a:2e:2c:bb:99";
        };

        k3s-rock5 = makeRock64K3sConfig {
          hostname = "k3s-rock5";
          selfIp = "192.168.20.114";
          serverIp = "192.168.20.120";
          macAddress = "02:76:67:0f:a0:b7";
        };

        k3s-rock6 = makeRock64K3sConfig {
          hostname = "k3s-rock6";
          selfIp = "192.168.20.115";
          serverIp = "192.168.20.120";
          macAddress = "02:6b:c5:f0:d7:6d";
        };

        k3s-quartza1 = makeQuartz64AK3sConfig {
          hostname = "k3s-quartza1";
          selfIp = "192.168.20.160";
          serverIp = "192.168.20.120";
        };

        k3s-rockpro1 = makeRockPro64K3sConfig {
          hostname = "k3s-rockpro1";
          selfIp = "192.168.20.180";
          serverIp = "192.168.20.120";
        };
      };

      deploy.nodes =
        let
          nodes = [
            { name = "redtail1"; ip = "192.168.20.23"; }
            { name = "whitestorm2"; ip = "192.168.20.27"; }
            { name = "garage1"; ip = "192.168.20.21"; }
            { name = "garage2"; ip = "192.168.20.28"; }
            { name = "garage3"; ip = "192.168.20.29"; }
            { name = "build2"; ip = "192.168.20.101"; }
            { name = "k3s1"; ip = "192.168.20.120"; }
            { name = "k3s2"; ip = "192.168.20.121"; }
            { name = "k3s3"; ip = "192.168.20.122"; }
            { name = "k3s4"; ip = "192.168.20.123"; }
            { name = "k3s5"; ip = "192.168.20.124"; }
            { name = "k3s6"; ip = "192.168.20.125"; }
            { name = "k3s-rock1"; ip = "192.168.20.20"; }
            { name = "k3s-rock2"; ip = "192.168.20.111"; }
            { name = "k3s-rock3"; ip = "192.168.20.112"; }
            { name = "k3s-rock4"; ip = "192.168.20.113"; }
            { name = "k3s-rock5"; ip = "192.168.20.114"; }
            { name = "k3s-rock6"; ip = "192.168.20.115"; }
            { name = "k3s-quartza1"; ip = "192.168.20.160"; }
            { name = "k3s-rockpro1"; ip = "192.168.20.180"; }
          ];
        in
        builtins.foldl'
          (acc: { name, ip }: acc // {
            ${name} = deployer {
              hostname = ip;
              configuration = self.nixosConfigurations.${name};
            };
          })
          { }
          nodes;
    };
}