commit 8f23cbca2e768211d9669a49c075976614a3b39d parent 5a873c38dad85d8ede9cd70e62a36daa611c3050 Author: MTRNord <mtrnord1@gmail.com> Date: Sun, 17 Sep 2023 14:28:37 +0200 Fix upstream Diffstat:
| M | nixos/worker-1/znc.nix | | | 2 | +- |
| A | nixos/worker-2/boot.nix | | | 34 | ++++++++++++++++++++++++++++++++++ |
| A | nixos/worker-2/configuration.nix | | | 358 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | nixos/worker-2/darlings.nix | | | 59 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | nixos/worker-2/hardware-configuration.nix | | | 73 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | nixos/worker-2/kubernetes.nix | | | 78 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
6 files changed, 603 insertions(+), 1 deletion(-)
diff --git a/nixos/worker-1/znc.nix b/nixos/worker-1/znc.nix @@ -37,7 +37,7 @@ upstreams = { "znc" = { servers = { - "localhost:58457" = { }; + "[::1]:58457" = { }; }; }; }; diff --git a/nixos/worker-2/boot.nix b/nixos/worker-2/boot.nix @@ -0,0 +1,34 @@ +{ lib, pkgs, config, ... }: +{ + boot = { + loader = { + systemd-boot = { + enable = true; + configurationLimit = 10; + editor = false; + }; + efi.canTouchEfiVariables = true; + }; + kernelParams = [ "ip=dhcp" ]; + initrd = { + network.enable = true; + luks.forceLuksSupportInInitrd = true; + network.ssh = { + enable = true; + # Defaults to 22. + port = 2222; + shell = "/bin/cryptsetup-askpass"; + # The key is generated using `ssh-keygen -t ed25519 -N "" -f /etc/secrets/initrd/ssh_host_ed25519_key` + # + # Stored in plain text on boot partition, so don't reuse your host + # keys. Also, make sure to use a boot loader with support for initrd + # secrets (e.g. systemd-boot), or this will be exposed in the nix store + # to unprivileged users. + hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ]; + # I'll just authorize all keys authorized post-boot. + authorizedKeys = config.users.users.marcel.openssh.authorizedKeys.keys; + }; + }; + }; + +} diff --git a/nixos/worker-2/configuration.nix b/nixos/worker-2/configuration.nix @@ -0,0 +1,358 @@ +# This is your system's configuration file. +# Use this to configure your system environment (it replaces /etc/nixos/configuration.nix) + +{ inputs, outputs, lib, config, pkgs, ... }: { + # You can import other NixOS modules here + imports = [ + # If you want to use modules your own flake exports (from modules/nixos): + # outputs.nixosModules.example + + # Or modules from other flakes (such as nixos-hardware): + # inputs.hardware.nixosModules.common-cpu-amd + inputs.hardware.nixosModules.common-pc-ssd + + inputs.impermanence.nixosModules.impermanence + + # You can also split up your configuration and import pieces of it here: + # ./users.nix + + # Import your generated (nixos-generate-config) hardware configuration + ./hardware-configuration.nix + + ../common/sops.nix + ../common/common.nix + ../common/server.nix + ./darlings.nix + ./boot.nix + + ../common/lib/shell.nix + ../common/lib/confd.nix + ../common/lib/fail2ban.nix + ../common/lib/podman.nix + + ./kubernetes.nix + ]; + + nixpkgs = { + # You can add overlays here + overlays = [ + # Add overlays your own flake exports (from overlays and pkgs dir): + outputs.overlays.additions + outputs.overlays.modifications + outputs.overlays.unstable-packages + + # You can also add overlays exported from other flakes: + # neovim-nightly-overlay.overlays.default + + # Or define it inline, for example: + # (final: prev: { + # hi = final.hello.overrideAttrs (oldAttrs: { + # patches = [ ./change-hello-to-hi.patch ]; + # }); + # }) + ]; + # Configure your nixpkgs instance + config = { + # Disable if you don't want unfree packages + allowUnfree = true; + + asterisk = { + withOpus = true; + }; + + permittedInsecurePackages = [ + "nodejs-16.20.2" + ]; + }; + }; + + nix = { + # This will add each flake input as a registry + # To make nix3 commands consistent with your flake + registry = lib.mapAttrs (_: value: { flake = value; }) inputs; + + # This will additionally add your inputs to the system's legacy channels + # Making legacy nix commands consistent as well, awesome! + nixPath = lib.mapAttrsToList (key: value: "${key}=${value.to.path}") config.nix.registry; + + settings = { + # Enable flakes and new 'nix' command + experimental-features = "nix-command flakes"; + # Deduplicate and optimize nix store + auto-optimise-store = true; + # Sandbox + sandbox = true; + # Build locally + trusted-substituters = [ ]; + substituters = [ ]; + }; + + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 60d"; + }; + + }; + + # Broken + systemd.network.wait-online.enable = false; + systemd.network = { + networks = { + "20-v6" = { + matchConfig = { + MACAddress = "96:00:02:44:cf:52"; + }; + address = [ + "49.13.24.105/32" + "2a01:4f8:c012:492::1/64" + ]; + routes = [ + { routeConfig.Gateway = "fe80::1"; } + { routeConfig = { Gateway = "172.31.1.1"; GatewayOnLink = true; }; } + + # prevent some local traffic Hetzner doesn't like + #{ routeConfig = { Destination = "172.16.0.0/12"; Type = "unreachable"; }; } + { routeConfig = { Destination = "192.168.0.0/16"; Type = "unreachable"; }; } + + # { routeConfig = { Destination = "10.0.0.0/8"; Type = "unreachable"; }; } + { routeConfig = { Destination = "fc00::/7"; Type = "unreachable"; }; } + + ]; + }; + }; + }; + + networking = { + hostName = "worker-2"; + enableIPv6 = true; + useNetworkd = true; + useDHCP = true; + # networkmanager.enable = true; + + nat = { + enable = true; + enableIPv6 = true; + externalInterface = "enp1s0"; + internalInterfaces = [ "wg0" ]; + }; + + nameservers = [ "8.8.8.8" "8.8.4.4" ]; + + wg-quick.interfaces = { + wg0 = { + address = [ "10.100.0.3/24" "fe99:13::3/64" ]; + listenPort = 51820; + privateKeyFile = config.sops.secrets."wireguard/worker-2/private_key".path; + table = "off"; + + peers = [ + # big one + { + publicKey = "M+OpQ/umgERHB+K6JJkszVChrRPqqYvMstbr28HRrSE="; + allowedIPs = [ + "0.0.0.0/0" + "::/0" + ]; + endpoint = "95.217.202.35:51820"; + } + # worker-1 + { + publicKey = "IGlPQDCrkWDPgzxSADbed/UFLxz93K+rRXXu9aa4+G8="; + allowedIPs = [ + "0.0.0.0/0" + "::/0" + ]; + endpoint = "49.13.24.105:51820"; + } + ]; + }; + + }; + + firewall = + let + blockedV4 = [ + "158.101.19.243" # full-text search scraper https://macaw.social/@angilly/109597402157254670 + "207.231.106.226" # fediverse.network / fedi.ninja + "45.81.20.80" # instances.social + "198.58.122.231" # fedimapper.tedivm.com + "142.93.3.121" # fedidb.org + "45.158.40.164" # fedi.buzz + "170.39.215.216" # fediverse.observer + "87.157.136.163" # fedi_stats + "94.31.103.67" # python/federation + "45.56.100.29" # scottherr? same as :5a13 + "173.230.137.240" # scottherr@mastodon.social + "138.37.89.34" + "104.21.80.126" # gangstalking.services + "172.67.181.16" # gangstalking.services + "198.98.54.220" # ryona.agency + "35.173.245.194" + "99.105.215.234" # public tl + "65.108.204.30" # unknown + "65.109.31.111" # @fediverse@mastodont.cat + "54.37.233.246" # fba.ryona.agency domain block scraper + "185.244.192.119" # mooneyed.de / drow.be / bka.li blocklist scraper + "23.24.204.110" # ryona tool fed.dembased.xyz / annihilation.social blocklist scraper + "187.190.192.31" # ryona tool unfediblockthefedi.now + "70.106.192.146" # blocklist scraper + # https://openai.com/gptbot-ranges.txt + "20.15.240.64/28" + "20.15.240.80/28" + "20.15.240.96/28" + "20.15.240.176/28" + "20.15.241.0/28" + "20.15.242.128/28" + "20.15.242.144/28" + "20.15.242.192/28" + "40.83.2.64/28" + ]; + blockedV6 = [ + "2003:cb:ff2c:2700::1/64" # fedi_stats + "2600:3c02::/64" # scottherr stats + "2600:3c03::/64" # unknown, tries public tl access + "2605:6400:10:1fe::1/64" # ryona.agency + "2a01:4f9:5a:1cc4::2" # @fediverse@mastodont.cat + "2604:a880:400:d1::1/64" # fedidb.org + "2a01:4f8:162:6027::1/64" # blocklist scraper + ]; + in + { + checkReversePath = "loose"; + trustedInterfaces = [ "tailscale0" "floating1" "wg0" ]; + enable = true; + allowPing = true; + allowedTCPPorts = [ + 22 # ssh + 51820 + 9962 + 9100 + ]; + + extraCommands = + builtins.concatStringsSep "\n" (builtins.map (ip: "iptables -A INPUT -s ${ip} -j DROP") blockedV4) + "\n" + + builtins.concatStringsSep "\n" (builtins.map (ip: "ip6tables -A INPUT -s ${ip} -j DROP") blockedV6) + "\n" + + '' + iptables -A nixos-fw -p tcp --source 10.245.0.0/16 -j nixos-fw-accept + ip6tables -A nixos-fw -p tcp --source fd00::/104 -j nixos-fw-accept + ''; + + extraStopCommands = + builtins.concatStringsSep "\n" (builtins.map (ip: "iptables -D INPUT -s ${ip} -j DROP") blockedV4) + "\n" + + builtins.concatStringsSep "\n" (builtins.map (ip: "ip6tables -D INPUT -s ${ip} -j DROP") blockedV6) + "\n" + + '' + iptables -D nixos-fw -p tcp --source 10.245.0.0/16 -j nixos-fw-accept + ip6tables -D nixos-fw -p tcp --source fd00::/104 -j nixos-fw-accept + ''; + + }; + }; + + # Configure your system-wide user settings (groups, etc), add more users as needed. + users = { + #mutableUsers = false; + users = { + marcel = { + isNormalUser = true; + passwordFile = config.sops.secrets.marcel_initial_password.path; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKUzC9NeEc4voBeAO7YuQ1ewRKCS2iar4Bcm4cKoNKUH mtrnord@nordgedanken.dev" + ]; + extraGroups = [ "wheel" ]; + shell = pkgs.zsh; + }; + + "root".passwordFile = config.sops.secrets.root_initial_password.path; + }; + }; + + # Restic Backup + services.restic.backups = { + storagebox = { + passwordFile = config.sops.secrets.backup_password.path; + paths = [ + "/persist" + ]; + repository = "sftp://u362507@u362507.your-storagebox.de:22//backups/worker-2"; + timerConfig = { + OnCalendar = "00:05"; + RandomizedDelaySec = "5h"; + }; + pruneOpts = [ + "--keep-daily 7" + "--keep-weekly 5" + "--keep-monthly 12" + ]; + initialize = true; + }; + }; + + security.acme.acceptTerms = true; + security.acme.defaults.email = "support@nordgedanken.dev"; + + services = { + bird2 = { + enable = true; + config = '' + router id 100.64.0.1; + debug protocols all; + + protocol device { + } + + protocol direct { + ipv4; + ipv6; + } + + protocol kernel { + ipv4 { + import all; + export all; + }; + } + + protocol kernel { + ipv6 { + import all; + export all; + }; + } + + protocol ospf v2 v4 { + ipv4 { + import all; + export all; + }; + graceful restart 1; + area 0 { + interface "wg0"; + }; + } + + protocol ospf v3 v6 { + ipv6 { + import all; + export all; + }; + graceful restart 1; + area 0 { + interface "wg0"; + }; + } + ''; + }; + + bird-lg = { + proxy = { + enable = true; + allowedIPs = [ "10.100.0.1" ]; + listenAddress = "10.100.0.3:8000"; + }; + }; + }; + + # https://nixos.wiki/wiki/FAQ/When_do_I_update_stateVersion + system.stateVersion = "23.05"; +} diff --git a/nixos/worker-2/darlings.nix b/nixos/worker-2/darlings.nix @@ -0,0 +1,59 @@ +{ lib, pkgs, config, ... }: +{ + # Darling Erasure + environment.persistence."/persist" = { + directories = [ + "/etc/nixos" + ]; + files = [ + "/etc/machine-id" + #"/etc/NIXOS" + "/etc/secrets/initrd/ssh_host_ed25519_key" + "/etc/secrets/initrd/ssh_host_ed25519_key.pub" + "/var/lib/sops-nix/key.txt" + ]; + }; + security.sudo.extraConfig = '' + # rollback results in sudo lectures after each reboot + Defaults lecture = never + ''; + # Note `lib.mkBefore` is used instead of `lib.mkAfter` here. + boot.initrd.postDeviceCommands = pkgs.lib.mkBefore '' + mkdir -p /mnt + + # We first mount the btrfs root to /mnt + # so we can manipulate btrfs subvolumes. + mount -o subvol=/ /dev/mapper/enc /mnt + + # While we're tempted to just delete /root and create + # a new snapshot from /root-blank, /root is already + # populated at this point with a number of subvolumes, + # which makes `btrfs subvolume delete` fail. + # So, we remove them first. + # + # /root contains subvolumes: + # - /root/var/lib/portables + # - /root/var/lib/machines + # + # I suspect these are related to systemd-nspawn, but + # since I don't use it I'm not 100% sure. + # Anyhow, deleting these subvolumes hasn't resulted + # in any issues so far, except for fairly + # benign-looking errors from systemd-tmpfiles. + btrfs subvolume list -o /mnt/root | + cut -f9 -d' ' | + while read subvolume; do + echo "deleting /$subvolume subvolume..." + btrfs subvolume delete "/mnt/$subvolume" + done && + echo "deleting /root subvolume..." && + btrfs subvolume delete /mnt/root + + echo "restoring blank /root subvolume..." + btrfs subvolume snapshot /mnt/root-blank /mnt/root + + # Once we're done rolling back to a blank snapshot, + # we can unmount /mnt and continue on the boot process. + umount /mnt + ''; +} diff --git a/nixos/worker-2/hardware-configuration.nix b/nixos/worker-2/hardware-configuration.nix @@ -0,0 +1,73 @@ +# Do not modify this file! It was generated by ‘nixos-generate-config’ +# and may be overwritten by future invocations. Please make changes +# to /etc/nixos/configuration.nix instead. +{ config, lib, pkgs, modulesPath, ... }: + +{ + imports = [ + (modulesPath + "/profiles/qemu-guest.nix") + ]; + + boot = { + initrd.availableKernelModules = [ "xhci_pci" "virtio_pci" "virtio_scsi" "usbhid" "sr_mod" ]; + initrd.kernelModules = [ ]; + kernelModules = [ ]; + extraModulePackages = [ ]; + }; + + fileSystems."/" = { + device = "/dev/disk/by-uuid/2dd0146f-acd1-4d76-a353-9f4cf7721972"; + fsType = "btrfs"; + options = [ "subvol=root" "compress=zstd" "noatime" ]; + }; + + boot.initrd.luks.devices."enc".device = "/dev/disk/by-uuid/56da9aee-dc91-4736-ae22-781e46ccb25e"; + + fileSystems = { + "/home" = { + device = "/dev/disk/by-uuid/2dd0146f-acd1-4d76-a353-9f4cf7721972"; + fsType = "btrfs"; + options = [ "subvol=home" "compress=zstd" "noatime" ]; + }; + + "/nix" = { + device = "/dev/disk/by-uuid/2dd0146f-acd1-4d76-a353-9f4cf7721972"; + fsType = "btrfs"; + options = [ "subvol=nix" "compress=zstd" "noatime" ]; + }; + + "/persist" = { + device = "/dev/disk/by-uuid/2dd0146f-acd1-4d76-a353-9f4cf7721972"; + fsType = "btrfs"; + options = [ "subvol=persist" "compress=zstd" "noatime" ]; + neededForBoot = true; + }; + + "/var/log" = { + device = "/dev/disk/by-uuid/2dd0146f-acd1-4d76-a353-9f4cf7721972"; + fsType = "btrfs"; + options = [ "subvol=log" "compress=zstd" "noatime" ]; + neededForBoot = true; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/840B-723D"; + fsType = "vfat"; + }; + }; + + swapDevices = [{ + device = "/var/lib/swapfile"; + size = 16 * 1024; + }]; + + # Enables DHCP on each ethernet and wireless interface. In case of scripted networking + # (the default) this is the recommended approach. When using systemd-networkd it's + # still possible to use this option, but it's recommended to use it in conjunction + # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`. + networking.useDHCP = lib.mkDefault true; + # networking.interfaces.enp1s0.useDHCP = lib.mkDefault true; + # networking.interfaces.enp7s0.useDHCP = lib.mkDefault true; + + nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux"; +} diff --git a/nixos/worker-2/kubernetes.nix b/nixos/worker-2/kubernetes.nix @@ -0,0 +1,78 @@ +{ lib, pkgs, config, ... }: +{ + environment.persistence."/persist" = { + directories = [ + "/var/lib/kubernetes/" + "/etc/kubernetes/" + "/var/lib/kubelet" + "/var/lib/etcd" + ]; + }; + + environment.systemPackages = with pkgs; [ + cri-o + kubernetes + iproute2 + ethtool + socat + cni + conntrack-tools + cri-tools + cilium-cli + k9s + ]; + + boot.kernelModules = [ + "iptable_nat" + "iptable_filter" + "xt_nat" + "br_netfilter" + "ip6table_mangle" + "ip6table_raw" + "ip6table_filter" + "ip6_tables" + ]; + + networking.firewall.allowedTCPPorts = [ + 4240 + 4244 + 4245 + 4250 + 10250 + ]; + networking.firewall.allowedUDPPorts = [ + 8473 + 51871 + ]; + + + services.kubernetes = { + apiserverAddress = "https://[2a01:4f9:4a:451c:2::5]:6443"; + masterAddress = "[2a01:4f9:4a:451c:2::5]"; + roles = [ "master" "node" ]; + proxy.enable = false; + addons.dns.enable = true; + easyCerts = false; + caFile = config.sops.secrets.kubernetes_ca_file.path; + dataDir = "/var/lib/kubelet"; + kubelet = { + clusterDns = lib.mkForce "10.96.0.10"; + cni.configDir = "/persist/kubernetes/cni"; + enable = true; + kubeconfig = { + server = "https://[2a01:4f9:4a:451c:2::5]:6443"; + }; + clientCaFile = config.sops.secrets.kubernetes_ca_client_file.path; + extraOpts = "--fail-swap-on=false --kubeconfig=/etc/kubernetes/kubelet.conf --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml"; + taints = { + "arm64" = { + key = "arch"; + value = "arm64"; + effect = "NoSchedule"; + }; + }; + }; + }; + services.kubernetes.flannel.enable = false; + services.kubernetes.kubelet.cni.packages = lib.mkForce [ pkgs.cni-plugins ]; +}