Add SD image pipeline, documentation overhaul, and fix module issues

- Add automatic SD image builds for Raspberry Pi via Forgejo Actions - Enable binfmt emulation on cryodev-main for aarch64 cross-builds - Add sd-image.nix module to cryodev-pi configuration - Create comprehensive docs/ structure with installation guides - Split installation docs into: first-install (server), reinstall, new-client (Pi) - Add lib/utils.nix and apps/rebuild from synix - Fix headplane module for new upstream API (tale/headplane) - Fix various module issues (mailserver stateVersion, option conflicts) - Add placeholder secrets.yaml files for both hosts - Remove old INSTRUCTIONS.md (content moved to docs/)
2026-03-11 08:41:58 +01:00 · 2026-03-11 08:41:58 +01:00 · 5ba78886d2
commit 5ba78886d2
parent a5261d8ff0
44 changed files with 3570 additions and 609 deletions
--- a/.forgejo/workflows/build-pi-image.yml
+++ b/.forgejo/workflows/build-pi-image.yml
@ -0,0 +1,97 @@
 name: Build Raspberry Pi SD Images
 on:
  push:
    branches:
      - main
    paths:
      - 'hosts/**'
      - 'modules/**'
      - 'templates/**'
      - 'flake.nix'
      - 'flake.lock'
      - 'constants.nix'
  workflow_dispatch:
 jobs:
  build-pi-images:
    runs-on: host
    strategy:
      matrix:
        # Add new Pi hosts to this list when created
        host: [cryodev-pi]
      fail-fast: false
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Build SD image for ${{ matrix.host }}
        run: |
          echo "Building SD image for: ${{ matrix.host }}"
          echo "This may take 30-60 minutes with emulation..."
          nix build .#nixosConfigurations.${{ matrix.host }}.config.system.build.sdImage \
            --extra-platforms aarch64-linux \
            --out-link result-${{ matrix.host }}
          IMAGE_PATH=$(find result-${{ matrix.host }} -name "*.img.zst" -type f | head -1)
          if [ -z "$IMAGE_PATH" ]; then
            echo "Error: No image found!"
            exit 1
          fi
          cp "$IMAGE_PATH" ./${{ matrix.host }}-sd-image.img.zst
          sha256sum ${{ matrix.host }}-sd-image.img.zst > ${{ matrix.host }}-sd-image.img.zst.sha256
          echo "Image size:"
          ls -lh ${{ matrix.host }}-sd-image.img.zst
      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.host }}-sd-image
          path: |
            ${{ matrix.host }}-sd-image.img.zst
            ${{ matrix.host }}-sd-image.img.zst.sha256
  create-release:
    needs: build-pi-images
    runs-on: host
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Download all artifacts
        uses: actions/download-artifact@v3
        with:
          path: artifacts/
      - name: Create Release and Upload
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          VERSION="v$(date +%Y-%m-%d)-$(git rev-parse --short HEAD)"
          # Create release via API
          curl -s -X POST \
            -H "Authorization: token ${GITHUB_TOKEN}" \
            -H "Content-Type: application/json" \
            -d "{\"tag_name\": \"${VERSION}\", \"name\": \"Pi Images ${VERSION}\", \"body\": \"Raspberry Pi SD card images. See docs for usage.\", \"draft\": false, \"prerelease\": false}" \
            "https://git.cryodev.xyz/api/v1/repos/${GITHUB_REPOSITORY}/releases" \
            -o release.json
          RELEASE_ID=$(jq -r '.id' release.json)
          echo "Release ID: $RELEASE_ID"
          # Upload all files
          for file in $(find artifacts -type f); do
            echo "Uploading: $(basename $file)"
            curl -s -X POST \
              -H "Authorization: token ${GITHUB_TOKEN}" \
              -H "Content-Type: application/octet-stream" \
              --data-binary @"$file" \
              "https://git.cryodev.xyz/api/v1/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=$(basename $file)"
          done
          echo "Done: https://git.cryodev.xyz/${GITHUB_REPOSITORY}/releases/tag/${VERSION}"
--- a/INSTRUCTIONS.md
+++ b/INSTRUCTIONS.md
@ -1,391 +0,0 @@
 # Server Setup Instructions / Server-Einrichtungsanleitung
 ---
 # 🇬🇧 English Instructions
 ## 1. Prerequisites
 Ensure you have the following tools installed on your local machine:
 -   `nix` (with flakes enabled)
 -   `sops`
 -   `age`
 -   `ssh`
 ## 2. DNS Configuration
 Configure the following DNS records for your domain `cryodev.xyz`:
 | Hostname | Type | Value | Purpose |
 |----------|------|-------|---------|
 | `@` | A | `<SERVER_IP>` | Main entry point |
 | `@` | AAAA | `<SERVER_IPV6>` | Main entry point (IPv6) |
 | `git` | CNAME | `@` | Forgejo |
 | `headscale` | CNAME | `@` | Headscale |
 | `headplane` | CNAME | `@` | Headplane |
 | `netdata` | CNAME | `@` | Netdata Monitoring |
 | `mail` | A | `<SERVER_IP>` | Mailserver |
 | `mail` | AAAA | `<SERVER_IPV6>` | Mailserver (IPv6) |
 | `@` | MX | `10 mail.cryodev.xyz.` | Mail delivery |
 | `@` | TXT | `"v=spf1 mx ~all"` | SPF Record |
 | `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC Record |
 ## 3. Secret Management (SOPS)
 This repository uses `sops-nix` to manage secrets encrypted with `age`, utilizing the SSH host keys of the servers.
 ### 3.1 Get Server Public Keys
 You need to convert the servers' SSH host public keys to age public keys.
 **For `cryodev-main`:**
 ```bash
 nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <MAIN_IP> | ssh-to-age'
 ```
 **For `cryodev-pi`:**
 ```bash
 nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <PI_IP> | ssh-to-age'
 ```
 ### 3.2 Configure `.sops.yaml`
 Edit the `.sops.yaml` file in the root of this repository. Add the age public keys to the `keys` section and ensure creation rules exist for both hosts.
 ```yaml
 keys:
  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t # Admin Key (Steffen)
  - &main_key age1... # cryodev-main Key
  - &pi_key age1...   # cryodev-pi Key
 creation_rules:
  - path_regex: hosts/cryodev-main/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *main_key
  - path_regex: hosts/cryodev-pi/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *pi_key
 ```
 ### 3.3 Generating Secret Values
 **Mailserver Passwords (for `cryodev-main`):**
 ```bash
 nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'
 ```
 **Headplane Secrets (for `cryodev-main`):**
 ```bash
 nix-shell -p openssl --run "openssl rand -hex 16"
 # Agent Pre-Authkey requires Headscale running:
 sudo headscale users create headplane-agent
 sudo headscale preauthkeys create --expiration 99y --reusable --user headplane-agent
 ```
 **Tailscale Auth Keys (for both hosts):**
 *Requires Headscale running on `cryodev-main`.*
 ```bash
 # For cryodev-main:
 sudo headscale preauthkeys create --expiration 99y --reusable --user default
 # For cryodev-pi:
 sudo headscale preauthkeys create --expiration 99y --reusable --user default
 ```
 **Netdata Child UUID (for `cryodev-pi`):**
 ```bash
 uuidgen
 ```
 **Forgejo Runner Token:**
 Get from Forgejo Admin Panel.
 ### 3.4 Creating Secrets Files
 **`hosts/cryodev-main/secrets.yaml`:**
 ```bash
 sops hosts/cryodev-main/secrets.yaml
 ```
 ```yaml
 mailserver:
    accounts:
        forgejo: "$2y$05$..."
        admin: "$2y$05$..."
 forgejo-runner:
    token: "..."
 headplane:
    cookie_secret: "..."
    agent_pre_authkey: "..."
 tailscale:
    auth-key: "..."
 ```
 **`hosts/cryodev-pi/secrets.yaml`:**
 ```bash
 sops hosts/cryodev-pi/secrets.yaml
 ```
 ```yaml
 tailscale:
    auth-key: "..."
 netdata:
    stream:
        child-uuid: "..." # Output from uuidgen
 ```
 ## 4. Initial Deployment (Bootstrap)
 Before the continuous deployment can take over, you must perform an initial deployment manually using the provided install script.
 ### 4.1 Prepare Target Machine
 1.  Boot into the NixOS Installation ISO.
 2.  Set a root password (for SSH): `passwd`.
 3.  Ensure internet connectivity.
 ### 4.2 Run Install Script
 From your local machine (where this repo is), copy the script to the target or run it directly if you can fetch it.
 **Method A: Copy Script via SSH**
 ```bash
 scp scripts/install.sh nixos@<TARGET_IP>:install.sh
 ssh nixos@<TARGET_IP>
 sudo -i
 chmod +x /home/nixos/install.sh
 ./home/nixos/install.sh -r <GIT_REPO_URL> -n <HOSTNAME>
 ```
 **Method B: Run on Target (if repo is public or reachable)**
 ```bash
 # On the target machine (as root)
 nix-shell -p git
 git clone <GIT_REPO_URL> /tmp/nixos
 cd /tmp/nixos
 bash scripts/install.sh -n <HOSTNAME>
 ```
 *Note: The script handles disk partitioning (via disko/script), hardware config generation, and installation.*
 ## 5. Continuous Deployment (CD)
 ### 5.1 cryodev-pi (Pull-based via Comin)
 The `cryodev-pi` host is configured to pull updates automatically via `comin`.
 1.  **Create Repository:** Create a new repository named `cryodev-server` on your Forgejo instance (`https://git.cryodev.xyz`).
 2.  **Push Configuration:** Push this entire NixOS configuration to the `main` branch of that repository.
 3.  **Comin URL:** The configuration expects the repository at: `https://git.cryodev.xyz/steffen/cryodev-server.git`.
 ### 5.2 cryodev-main (Push-based via Forgejo Actions)
 The main server is deployed via a Forgejo Action.
 1.  **Generate SSH Key:**
    ```bash
    ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
    ```
 2.  **Add Public Key:** Add the content of `deploy_key.pub` to `/root/.ssh/authorized_keys` on `cryodev-main`.
 3.  **Add Secret:** Add the content of `deploy_key` (private key) as a secret named `DEPLOY_SSH_KEY` in your Forgejo repository settings.
 ## 6. Creating New Hosts (Templates)
 To quickly bootstrap a new host configuration, you can use the provided templates.
 1.  **Copy Template:**
    ```bash
    # For a Raspberry Pi:
    cp -r templates/raspberry-pi hosts/new-pi-name
    # For a generic x86 server:
    cp -r templates/generic-server hosts/new-server-name
    ```
 2.  **Adjust Configuration:**
    *   **Hostname:** Edit `hosts/new-name/networking.nix`.
    *   **Flake:** Register the new host in `flake.nix` under `nixosConfigurations`.
    *   **Constants:** Add IP and ports to `constants.nix`.
    *   **Secrets:** Add keys to `.sops.yaml` and create `hosts/new-name/secrets.yaml`.
 ---
 # 🇩🇪 Deutsche Anleitung
 ## 1. Voraussetzungen
 Stellen Sie sicher, dass folgende Tools lokal installiert sind:
 -   `nix` (mit Flakes)
 -   `sops`
 -   `age`
 -   `ssh`
 -   `ssh-to-age`
 -   `uuidgen`
 ## 2. DNS-Konfiguration
 Richten Sie folgende DNS-Einträge für `cryodev.xyz` ein:
 | Hostname | Typ | Wert | Zweck |
 |----------|-----|------|-------|
 | `@` | A | `<SERVER_IP>` | Hauptserver |
 | `@` | AAAA | `<SERVER_IPV6>` | Hauptserver (IPv6) |
 | `git` | CNAME | `@` | Forgejo |
 | `headscale` | CNAME | `@` | Headscale |
 | `headplane` | CNAME | `@` | Headplane |
 | `netdata` | CNAME | `@` | Netdata Monitoring |
 | `mail` | A | `<SERVER_IP>` | Mailserver |
 | `mail` | AAAA | `<SERVER_IPV6>` | Mailserver (IPv6) |
 | `@` | MX | `10 mail.cryodev.xyz.` | Mail-Empfang |
 | `@` | TXT | `"v=spf1 mx ~all"` | SPF-Record |
 | `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC-Record |
 ## 3. Verwaltung von Geheimnissen (SOPS)
 Dieses Repository nutzt `sops-nix` mit den SSH-Host-Keys der Server.
 ### 3.1 Public Keys abrufen
 **Für `cryodev-main`:**
 ```bash
 nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <MAIN_IP> | ssh-to-age'
 ```
 **Für `cryodev-pi`:**
 ```bash
 nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <PI_IP> | ssh-to-age'
 ```
 ### 3.2 `.sops.yaml` konfigurieren
 Bearbeiten Sie `.sops.yaml` und fügen Sie die Keys sowie Regeln für beide Hosts hinzu:
 ```yaml
 keys:
  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t # Admin Key (Steffen)
  - &main_key age1... # cryodev-main Key
  - &pi_key age1...   # cryodev-pi Key
 creation_rules:
  - path_regex: hosts/cryodev-main/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *main_key
  - path_regex: hosts/cryodev-pi/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *pi_key
 ```
 ### 3.3 Werte generieren
 **Mailserver:** `mkpasswd -sm bcrypt`
 **Headplane:** `openssl rand -hex 16`
 **Netdata UUID:** `uuidgen`
 **Tailscale Auth Keys (auf `cryodev-main`):**
 ```bash
 sudo headscale preauthkeys create --expiration 99y --reusable --user default
 ```
 ### 3.4 Secrets-Dateien erstellen
 **`hosts/cryodev-main/secrets.yaml`:**
 ```bash
 sops hosts/cryodev-main/secrets.yaml
 ```
 ```yaml
 mailserver:
    accounts:
        forgejo: "$2y$05$..."
        admin: "$2y$05$..."
 forgejo-runner:
    token: "..."
 headplane:
    cookie_secret: "..."
    agent_pre_authkey: "..."
 tailscale:
    auth-key: "..."
 ```
 **`hosts/cryodev-pi/secrets.yaml`:**
 ```bash
 sops hosts/cryodev-pi/secrets.yaml
 ```
 ```yaml
 tailscale:
    auth-key: "..."
 netdata:
    stream:
        child-uuid: "..." # Output von uuidgen
 ```
 ## 4. Erstes Deployment (Bootstrap)
 Bevor das automatische Deployment funktionieren kann, müssen Sie das System einmal manuell mit dem Installationsskript installieren.
 ### 4.1 Zielmaschine vorbereiten
 1.  Booten Sie das NixOS Installations-ISO.
 2.  Setzen Sie ein Root-Passwort: `passwd`.
 3.  Stellen Sie eine Internetverbindung her.
 ### 4.2 Install-Script ausführen
 Kopieren Sie das Skript von Ihrem lokalen Rechner auf das Zielsystem.
 **Methode A: Per SCP**
 ```bash
 scp scripts/install.sh nixos@<TARGET_IP>:install.sh
 ssh nixos@<TARGET_IP>
 sudo -i
 chmod +x /home/nixos/install.sh
 ./home/nixos/install.sh -r <GIT_REPO_URL> -n <HOSTNAME>
 ```
 **Methode B: Direkt auf dem Ziel (bei öffentlichem/erreichbarem Repo)**
 ```bash
 # Auf der Zielmaschine (als root)
 nix-shell -p git
 git clone <GIT_REPO_URL> /tmp/nixos
 cd /tmp/nixos
 bash scripts/install.sh -n <HOSTNAME>
 ```
 *Hinweis: Das Skript kümmert sich um Partitionierung, Hardware-Config und Installation.*
 ## 5. Continuous Deployment (CD)
 ### 5.1 cryodev-pi (Pull-basiert via Comin)
 Der Host `cryodev-pi` zieht Updates automatisch via `comin`.
 1.  **Repository erstellen:** Erstellen Sie ein Repository namens `cryodev-server` auf `https://git.cryodev.xyz`.
 2.  **Konfiguration pushen:** Pushen Sie diese Konfiguration in den `main`-Branch.
 3.  **Comin URL:** `https://git.cryodev.xyz/steffen/cryodev-server.git`.
 ### 5.2 cryodev-main (Push-basiert via Forgejo Actions)
 Der Hauptserver wird über eine Forgejo Action deployt.
 1.  **SSH Key generieren:**
    ```bash
    ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
    ```
 2.  **Public Key hinzufügen:** Inhalt von `deploy_key.pub` in `/root/.ssh/authorized_keys` auf `cryodev-main` eintragen.
 3.  **Secret hinzufügen:** Inhalt von `deploy_key` (Private Key) als Secret `DEPLOY_SSH_KEY` im Forgejo-Repository hinterlegen.
 ## 6. Neue Hosts erstellen (Templates)
 Um schnell eine neue Host-Konfiguration zu erstellen, können Sie die bereitgestellten Templates nutzen.
 1.  **Template kopieren:**
    ```bash
    # Für einen Raspberry Pi:
    cp -r templates/raspberry-pi hosts/neuer-pi-name
    # Für einen generischen x86 Server:
    cp -r templates/generic-server hosts/neuer-server-name
    ```
 2.  **Konfiguration anpassen:**
    *   **Hostname:** Bearbeiten Sie `hosts/neuer-name/networking.nix`.
    *   **Flake:** Registrieren Sie den neuen Host in `flake.nix` unter `nixosConfigurations`.
    *   **Constants:** Fügen Sie IP und Ports in `constants.nix` hinzu.
    *   **Secrets:** Fügen Sie Keys zu `.sops.yaml` hinzu und erstellen Sie `hosts/neuer-name/secrets.yaml`.
--- a/README.md
+++ b/README.md
@ -1,147 +1,100 @@
-# cryodev-server NixOS Configuration
+# cryodev NixOS Configuration
-This repository contains the declarative NixOS configuration for the **cryodev** infrastructure, managed using **Nix Flakes**. It defines a robust, secure, and self-hosted environment spanning a main server and client devices.
+Declarative NixOS infrastructure for the **cryodev** environment, managed with Nix Flakes.
---
+## Quick Start
-# 🇬🇧 English Description
+```bash
 # Clone repository
 git clone https://git.cryodev.xyz/steffen/cryodev-server.git
 cd cryodev-server
-## Overview
+# Check configuration
 nix flake check
-The infrastructure is designed around a central server (`cryodev-main`) and satellite clients (e.g., `cryodev-pi`). It leverages modern DevOps practices including Infrastructure as Code (IaC), GitOps, and Mesh VPNs.
+# Build a host
 nix build .#nixosConfigurations.cryodev-main.config.system.build.toplevel
 ```
-## Key Features & Architecture
+## Hosts
-### 🖥️ Hosts
+| Host | Architecture | Deployment | Description |
-*   **`cryodev-main` (x86_64 Server)**: The core infrastructure hub.
+|------|--------------|------------|-------------|
-*   **`cryodev-pi` (Raspberry Pi 4)**: A remote client/worker node.
+| `cryodev-main` | x86_64 | Push (deploy-rs) | Main server |
 | `cryodev-pi` | aarch64 | Pull (Comin) | Raspberry Pi client |
-### 🚀 Continuous Deployment (CD)
+## Services
 We utilize different deployment strategies optimized for each host type:
 *   **Push-based (Server):** The main server is deployed via **Forgejo Actions** using **[deploy-rs](https://github.com/serokell/deploy-rs)**. This ensures immediate updates and robust rollbacks in case of failure.
 *   **Pull-based (Client):** The Raspberry Pi uses **[Comin](https://github.com/nlewo/comin)** to periodically poll the Git repository and apply updates automatically. This is ideal for devices behind NAT or with unstable connections.
-### 🌐 Networking (Tailscale & Headscale)
+| Service | Domain | Description |
-*   **Self-hosted VPN:** We run **Headscale**, an open-source implementation of the Tailscale control server.
+|---------|--------|-------------|
-*   **Headplane:** A web frontend for managing Headscale users and routes.
+| Headscale | `headscale.cryodev.xyz` | Self-hosted Tailscale server |
-*   **Mesh Network:** All hosts are connected via a secure, private WireGuard mesh network.
+| Headplane | `headplane.cryodev.xyz` | Headscale web UI |
-*   **MagicDNS:** Automatic DNS resolution for devices within the tailnet.
+| Forgejo | `git.cryodev.xyz` | Git hosting with CI/CD |
 | Netdata | `netdata.cryodev.xyz` | Monitoring dashboard |
 | Mail | `mail.cryodev.xyz` | Email (Postfix/Dovecot) |
-### 📊 Monitoring (Netdata)
+## Raspberry Pi SD Images
 *   **Parent/Child Streaming:** The main server acts as a centralized Netdata parent node.
 *   **Distributed Monitoring:** `cryodev-pi` (and other clients) stream their metrics securely over the Tailscale VPN to the parent node.
 *   **Alerting:** Integrated with the mailserver to send health alerts.
-### 📧 Mail Services
+SD card images for Raspberry Pi clients are **built automatically** on every push to `main`.
 *   **NixOS Mailserver:** A fully functional mail stack (Postfix/Dovecot).
 *   **Integration:** Used by internal services (Forgejo, Netdata) to send notifications.
 *   **Security:** SPF, DKIM, and DMARC configured for `cryodev.xyz`.
-### 🛠️ Development & Productivity
+Download from: [Releases](https://git.cryodev.xyz/steffen/cryodev-server/releases)
 *   **Forgejo:** Self-hosted Git service (fork of Gitea) with built-in CI/CD Actions.
 *   **Forgejo Runner:** Self-hosted runners executing the CI/CD pipelines.
 *   **Neovim:** A fully pre-configured Neovim environment (aliased as `v`) available system-wide via a custom NixOS module.
 *   **Secret Management:** **[sops-nix](https://github.com/Mic92/sops-nix)** encrypts secrets using Age and SSH host keys, ensuring no sensitive data is committed in plain text.
 *   **Templates:** Ready-to-use NixOS configurations for quickly bootstrapping new clients.
    *   `#raspberry-pi`: Template for Raspberry Pi 4 clients.
    *   `#generic-server`: Template for generic x86_64 servers.
 *   **Bootstrap Script:** An `install.sh` script automates disk partitioning (via disko) and system installation for new hosts.
-## 🚧 Roadmap & Missing Features
+```bash
 # Flash to SD card
 zstd -d cryodev-pi-sd-image.img.zst
 sudo dd if=cryodev-pi-sd-image.img of=/dev/sdX bs=4M status=progress
 ```
-### BioSafe Gateway (Dual Ethernet)
+See [Adding a new Raspberry Pi](docs/getting-started/new-client.md) for the full workflow.
 The Raspberry Pi hosts utilize a custom board with two Ethernet ports:
 *   **WAN:** Standard Internet connection.
 *   **LAN (`eth1`):** A dedicated local connection managed specifically by the **BioSafe Gateway App**.
 *   *Status:* The network configuration logic and the integration of the controlling app are currently missing.
-### Closed Source Integration
+## Documentation
 The **BioSafe Gateway App** is closed source.
 *   It needs to be added as a **private Flake input**.
 *   Authentication mechanism (e.g., access tokens via Secrets) to fetch this private input during the build process is not yet implemented.
-### SD Card Image Pipeline
+Full documentation is available in the [`docs/`](docs/index.md) directory:
-Currently, the Pi requires manual setup.
+
-*   *Goal:* A CI/CD pipeline that builds a fully configured, flashable SD card image.
+- [Prerequisites](docs/getting-started/prerequisites.md)
-*   *Usage:* Download image -> Flash to SD -> Insert in Pi -> Boot & Auto-connect.
+- [New Raspberry Pi Client](docs/getting-started/new-client.md)
 - [SD Image Reference](docs/getting-started/sd-image.md)
 - [Server Installation](docs/getting-started/first-install.md)
 - [Reinstallation](docs/getting-started/reinstall.md)
 - [Services](docs/services/)
 - [Deployment](docs/deployment/cd.md)
 ## Directory Structure
 *   `flake.nix`: The entry point defining inputs (dependencies) and outputs (system configs).
 *   `hosts/`: Configuration specific to each machine.
 *   `modules/`: Reusable NixOS modules (custom or imported).
 *   `pkgs/`: Custom packages.
 *   `constants.nix`: Central source of truth for IPs, ports, and domains.
 *   `INSTRUCTIONS.md`: Detailed setup guide (DNS, SOPS, Initial Deployment).
 *   `AGENTS.md`: Guidelines for AI agents working on this repository.
---
+```
 .
 ├── flake.nix           # Flake entry point
 ├── constants.nix       # Central configuration
 ├── hosts/              # Host configurations
 ├── modules/            # Reusable NixOS modules
 ├── pkgs/               # Custom packages
 ├── overlays/           # Nixpkgs overlays
 ├── templates/          # Host templates
 ├── scripts/            # Helper scripts
 ├── apps/               # Nix apps (rebuild)
 ├── lib/                # Helper functions
 └── docs/               # Documentation
 ```
-# 🇩🇪 Deutsche Beschreibung
+## Commands
-## Überblick
+```bash
 # Format code
 nix fmt
-Die Infrastruktur ist um einen zentralen Server (`cryodev-main`) und Satelliten-Clients (z.B. `cryodev-pi`) herum aufgebaut. Sie nutzt moderne DevOps-Praktiken wie Infrastructure as Code (IaC), GitOps und Mesh-VPNs.
+# Run checks
 nix flake check
-## Hauptfunktionen & Architektur
+# Update dependencies
 nix flake update
-### 🖥️ Hosts
+# Enter dev shell
-*   **`cryodev-main` (x86_64 Server)**: Der zentrale Infrastruktur-Knoten.
+nix develop
 *   **`cryodev-pi` (Raspberry Pi 4)**: Ein entfernter Client/Worker-Node.
-### 🚀 Continuous Deployment (CD)
+# Build Pi SD image locally
-Wir verwenden unterschiedliche Deployment-Strategien, optimiert für den jeweiligen Host-Typ:
+nix build .#nixosConfigurations.cryodev-pi.config.system.build.sdImage
-*   **Push-basiert (Server):** Der Hauptserver wird über **Forgejo Actions** mittels **[deploy-rs](https://github.com/serokell/deploy-rs)** aktualisiert. Dies garantiert sofortige Updates und robuste Rollbacks bei Fehlern.
+```
 *   **Pull-basiert (Client):** Der Raspberry Pi nutzt **[Comin](https://github.com/nlewo/comin)**, um das Git-Repository periodisch abzufragen und Updates automatisch anzuwenden. Ideal für Geräte hinter NAT oder mit instabilen Verbindungen.
-### 🌐 Netzwerk (Tailscale & Headscale)
+## License
 *   **Self-hosted VPN:** Wir betreiben **Headscale**, eine Open-Source-Implementierung des Tailscale-Kontrollservers.
 *   **Headplane:** Ein Web-Frontend zur Verwaltung von Headscale-Benutzern und Routen.
 *   **Mesh-Netzwerk:** Alle Hosts sind über ein sicheres, privates WireGuard-Mesh-Netzwerk verbunden.
 *   **MagicDNS:** Automatische Namensauflösung für Geräte innerhalb des Tailnets.
-### 📊 Monitoring (Netdata)
+Private repository.
 *   **Parent/Child Streaming:** Der Hauptserver agiert als zentraler Netdata Parent-Node.
 *   **Verteiltes Monitoring:** `cryodev-pi` (und andere Clients) streamen ihre Metriken sicher über das Tailscale-VPN an den Parent-Node.
 *   **Alarmierung:** Integriert mit dem Mailserver zum Versenden von Gesundheitswarnungen.
 ### 📧 Mail-Dienste
 *   **NixOS Mailserver:** Ein voll funktionsfähiger Mail-Stack (Postfix/Dovecot).
 *   **Integration:** Wird von internen Diensten (Forgejo, Netdata) für Benachrichtigungen genutzt.
 *   **Sicherheit:** SPF, DKIM und DMARC sind für `cryodev.xyz` konfiguriert.
 ### 🛠️ Entwicklung & Produktivität
 *   **Forgejo:** Self-hosted Git-Dienst (Gitea-Fork) mit integrierten CI/CD Actions.
 *   **Forgejo Runner:** Eigene Runner, die die CI/CD-Pipelines ausführen.
 *   **Neovim:** Eine vollständig vorkonfigurierte Neovim-Umgebung (Alias `v`), die systemweit über ein eigenes NixOS-Modul bereitgestellt wird.
 *   **Secret Management:** **[sops-nix](https://github.com/Mic92/sops-nix)** verschlüsselt Geheimnisse mittels Age und SSH-Host-Keys, sodass keine sensiblen Daten im Klartext gespeichert werden.
 *   **Templates:** Vorgefertigte NixOS-Konfigurationen zum schnellen Aufsetzen neuer Clients.
    *   `#raspberry-pi`: Template für Raspberry Pi 4 Clients.
    *   `#generic-server`: Template für generische x86_64 Server.
 *   **Bootstrap Skript:** Ein `install.sh` Skript automatisiert die Disk-Partitionierung (via disko) und Systeminstallation für neue Hosts.
 ## 🚧 Roadmap & Fehlende Funktionen
 ### BioSafe Gateway (Dual-Ethernet)
 Die Raspberry Pi Hosts nutzen ein spezielles Board mit zwei Ethernet-Anschlüssen:
 *   **WAN:** Normale Internetverbindung.
 *   **LAN (`eth1`):** Eine dedizierte lokale Verbindung, die spezifisch durch die **BioSafe Gateway App** verwaltet wird.
 *   *Status:* Die Logik für die Netzwerkkonfiguration und die Integration der steuernden App fehlen aktuell.
 ### Integration von Closed Source
 Die **BioSafe Gateway App** ist Closed Source.
 *   Sie muss als **privater Flake Input** hinzugefügt werden.
 *   Mechanismen zur Authentifizierung (z.B. Access Tokens via Secrets) für das Abrufen dieses privaten Inputs während des Build-Prozesses sind noch nicht implementiert.
 ### Pipeline für SD-Karten-Images
 Momentan erfordert der Pi eine manuelle Einrichtung.
 *   *Ziel:* Eine CI/CD-Pipeline, die ein fertig konfiguriertes, flashbares SD-Karten-Image baut.
 *   *Nutzung:* Image herunterladen -> auf SD flashen -> in Pi stecken -> booten & automatisch verbinden.
 ## Verzeichnisstruktur
 *   `flake.nix`: Der Einstiegspunkt, der Inputs (Abhängigkeiten) und Outputs (Systemkonfigurationen) definiert.
 *   `hosts/`: Maschinenspezifische Konfigurationen.
 *   `modules/`: Wiederverwendbare NixOS-Module (eigene oder importierte).
 *   `pkgs/`: Benutzerdefinierte Pakete.
 *   `constants.nix`: Zentrale Quelle für IPs, Ports und Domains.
 *   `INSTRUCTIONS.md`: Detaillierte Einrichtungsanleitung (DNS, SOPS, Initial Deployment).
 *   `AGENTS.md`: Richtlinien für KI-Agenten, die an diesem Repository arbeiten.
--- a/apps/rebuild/default.nix
+++ b/apps/rebuild/default.nix
@ -0,0 +1,30 @@
 {
  writeShellApplication,
  coreutils,
  gnugrep,
  gnused,
  home-manager,
  hostname,
  nix,
  nixos-rebuild,
  ...
 }:
 let
  name = "rebuild";
  text = builtins.readFile ./${name}.sh;
 in
 writeShellApplication {
  inherit name text;
  meta.mainProgram = name;
  runtimeInputs = [
    coreutils
    gnugrep
    gnused
    home-manager
    hostname
    nix
    nixos-rebuild
  ];
 }
--- a/apps/rebuild/rebuild.sh
+++ b/apps/rebuild/rebuild.sh
@ -0,0 +1,246 @@
 # NixOS and standalone Home Manager rebuild script
 # Defaults
 FLAKE_PATH="$HOME/.config/nixos" # Default flake path
 HOME_USER="$(whoami)"            # Default username. Used to identify the Home Manager configuration
 NIXOS_HOST="$(hostname)"         # Default hostname. Used to identify the NixOS and Home Manager configuration
 BUILD_HOST=""                    # Default build host. Empty means localhost
 TARGET_HOST=""                   # Default target host. Empty means localhost
 UPDATE=0                         # Default to not update flake repositories
 UPDATE_INPUTS=""                 # Default list of inputs to update. Empty means all
 ROLLBACK=0                       # Default to not rollback
 SHOW_TRACE=0                     # Default to not show detailed error messages
 # Function to display the help message
 Help() {
  echo "Wrapper script for 'nixos-rebuild switch' and 'home-manager switch' commands."
  echo "Usage: rebuild <command> [OPTIONS]"
  echo
  echo "Commands:"
  echo "  nixos                  Rebuild NixOS configuration"
  echo "  home                   Rebuild Home Manager configuration"
  echo "  all                    Rebuild both NixOS and Home Manager configurations"
  echo "  help                   Show this help message"
  echo
  echo "Options (for NixOS and Home Manager):"
  echo "  -H, --host <host>      Specify the hostname (as in 'nixosConfiguraions.<host>'). Default: $NIXOS_HOST"
  echo "  -p, --path <path>      Set the path to the flake directory. Default: $FLAKE_PATH"
  echo "  -U, --update [inputs]  Update all flake inputs. Optionally provide comma-separated list of inputs to update instead."
  echo "  -r, --rollback         Don't build the new configuration, but use the previous generation instead"
  echo "  -t, --show-trace       Show detailed error messages"
  echo
  echo "NixOS only options:"
  echo "  -B, --build-host <user@example.com>   Use a remote host for building the configuration via SSH"
  echo "  -T, --target-host <user@example.com>  Deploy the configuration to a remote host via SSH. If '--host' is specified, it will be used as the target host."
  echo
  echo "Home Manager only options:"
  echo "  -u, --user <user>      Specify the username (as in 'homeConfigurations.<user>@<host>'). Default: $HOME_USER"
 }
 # Function to handle errors
 error() {
  echo "Error: $1"
  exit 1
 }
 # Function to rebuild NixOS configuration
 Rebuild_nixos() {
  local FLAKE="$FLAKE_PATH#$NIXOS_HOST"
  # Construct rebuild command
  local CMD=("nixos-rebuild" "switch" "--sudo")
  [[ -n "$TARGET_HOST" || -n "$BUILD_HOST" ]] && CMD+=("--ask-sudo-password")
  CMD+=("--flake" "$FLAKE")
  [ "$ROLLBACK" = 1 ] && CMD+=("--rollback")
  [ "$SHOW_TRACE" = 1 ] && CMD+=("--show-trace")
  [ -n "$BUILD_HOST" ] && CMD+=("--build-host" "$BUILD_HOST")
  if [ "$NIXOS_HOST" != "$(hostname)" ] && [ -z "$TARGET_HOST" ]; then
    TARGET_HOST="$NIXOS_HOST"
    echo "Using '$TARGET_HOST' as target host."
  fi
  [ -n "$TARGET_HOST" ] && CMD+=("--target-host" "$TARGET_HOST")
  # Rebuild NixOS configuration
  if [ "$ROLLBACK" = 0 ]; then 
    echo "Rebuilding NixOS configuration '$FLAKE'..." 
  else
    echo "Rolling back to last NixOS generation..."
  fi
  echo "Executing command: ${CMD[*]}"
  "${CMD[@]}" || error "NixOS rebuild failed"
  echo "NixOS rebuild completed successfully."
 }
 # Function to rebuild Home Manager configuration
 Rebuild_home() {
  local FLAKE="$FLAKE_PATH#$HOME_USER@$NIXOS_HOST"
  if [ -n "$BUILD_HOST" ] || [ -n "$TARGET_HOST" ]; then
    error "Remote building is not supported for Home Manager."
  fi
  # Construct rebuild command
  local CMD=()
  if [ "$ROLLBACK" = 1 ]; then
    local rollback_path
    rollback_path=$(home-manager generations | sed -n '2p' | grep -o '/nix/store[^ ]*')
    CMD+=("$rollback_path/activate")
  else
    CMD=("home-manager" "switch" "--flake" "$FLAKE")
    [ "$SHOW_TRACE" = 1 ] && CMD+=("--show-trace")
  fi
  # Rebuild Home Manager configuration
  if [ "$ROLLBACK" = 0 ]; then 
    echo "Rebuilding Home Manager configuration '$FLAKE'..." 
  else
    echo "Rolling back to last Home Manager generation..."
  fi
  echo "Executing command: ${CMD[*]}"
  "${CMD[@]}" || error "Home Manager rebuild failed"
  echo "Home Manager rebuild completed successfully."
 }
 # Function to Update flake repositories
 Update() {
  echo "Updating flake inputs..."
  # Construct update command as an array
  local CMD=("nix" "flake" "update" "--flake" "$FLAKE_PATH")
  if [ -n "$UPDATE_INPUTS" ]; then
    # Split comma-separated inputs and pass them to nix flake update
    IFS=',' read -ra INPUTS <<< "$UPDATE_INPUTS"
    for input in "${INPUTS[@]}"; do
      CMD+=("$input")
    done
  fi
  echo "Executing command: ${CMD[*]}"
  "${CMD[@]}" || error "Failed to update flake repositories"
  echo "Flake repositories updated successfully."
 }
 # Parse command-line options
 if [[ -z "${1:-}" ]]; then
  echo "Error: No command specified. Printing help page."
  Help
  exit 1
 fi
 COMMAND=$1
 shift
 # Handle help command early
 if [ "$COMMAND" = "help" ] || [ "$COMMAND" = "--help" ] || [ "$COMMAND" = "-h" ]; then
  Help
  exit 0
 fi
 while [ $# -gt 0 ]; do
  case "${1:-}" in
    -H|--host)
      if [ -n "${2:-}" ]; then
        NIXOS_HOST="$2"
        shift 2
      else
        error "-H|--host option requires an argument"
      fi
      ;;
    -u|--user)
      if [ -n "${2:-}" ]; then
        HOME_USER="$2"
        shift 2
      else
        error "-u|--user option requires an argument"
      fi
      ;;
    -p|--path)
      if [ -n "${2:-}" ]; then
        FLAKE_PATH="$2"
        shift 2
      else
        error "-p|--path option requires an argument"
      fi
      ;;
    -U|--update)
      UPDATE=1
      # Check if next argument is a non-option
      if [ $# -gt 1 ] && [ "${2#-}" = "${2:-}" ]; then
        UPDATE_INPUTS="$2"
        shift 2
      else
        shift
      fi
      ;;
    -r|--rollback)
      ROLLBACK=1
      shift
      ;;
    -t|--show-trace)
      SHOW_TRACE=1
      shift
      ;;
    -B|--build-host)
      if [ -n "${2:-}" ]; then
        BUILD_HOST="$2"
        shift 2
      else
        error "-B|--build-host option requires an argument"
      fi
      ;;
    -T|--target-host)
      if [ -n "${2:-}" ]; then
        TARGET_HOST="$2"
        shift 2
      else
        error "-T|--target-host option requires an argument"
      fi
      ;;
    *)
      echo "Error: Unknown option '$1'"
      Help
      exit 1
      ;;
  esac
 done
 # Check if script is run with sudo
 if [ "$EUID" -eq 0 ]; then
  error "Do not run this script with sudo."
 fi
 # Check if flake path exists
 if [ ! -d "$FLAKE_PATH" ]; then
  error "Flake path '$FLAKE_PATH' does not exist"
 fi
 # Ignore trailing slash in flake path
 FLAKE_PATH="${FLAKE_PATH%/}"
 # Check if flake.nix exists
 if [ ! -f "$FLAKE_PATH/flake.nix" ]; then
  error "flake.nix does not exist in '$FLAKE_PATH'"
 fi
 # Execute updates and rebuilds based on the command
 [ "$UPDATE" = 1 ] && Update
 case "$COMMAND" in
  nixos)
    Rebuild_nixos
    ;;
  home)
    Rebuild_home
    ;;
  all)
    Rebuild_nixos
    Rebuild_home
    ;;
  *)
    echo "Error: Unknown command '$COMMAND'"
    echo "Printing help page:"
    Help
    exit 1
    ;;
 esac
--- a/docs/deployment/cd.md
+++ b/docs/deployment/cd.md
@ -0,0 +1,174 @@
 # Continuous Deployment
 The cryodev infrastructure uses two deployment strategies optimized for different host types.
 ## Overview
 | Host | Strategy | Tool | Trigger |
 |------|----------|------|---------|
 | `cryodev-main` | Push-based | deploy-rs | Git push via Forgejo Actions |
 | `cryodev-pi` | Pull-based | Comin | Periodic polling |
 ## Push-based Deployment (cryodev-main)
 ### How It Works
 1. Developer pushes to `main` branch
 2. Forgejo Actions workflow triggers
 3. `deploy-rs` connects via SSH and deploys
 ### Setup
 #### 1. Generate Deploy Key
 ```bash
 ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
 ```
 #### 2. Add Public Key to Server
 On `cryodev-main`:
 ```bash
 echo "PUBLIC_KEY_CONTENT" >> /root/.ssh/authorized_keys
 ```
 #### 3. Add Private Key to Forgejo
 1. Go to Repository Settings > Secrets
 2. Add secret named `DEPLOY_SSH_KEY`
 3. Paste the private key content
 #### 4. Workflow Configuration
 `.forgejo/workflows/deploy.yaml`:
 ```yaml
 name: Deploy
 on:
  push:
    branches: [main]
 jobs:
  check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: cachix/install-nix-action@v24
      - run: nix flake check
  deploy:
    needs: check
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: cachix/install-nix-action@v24
      - name: Setup SSH
        env:
          SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh
          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          ssh-keyscan cryodev-main >> ~/.ssh/known_hosts
      - name: Deploy
        run: nix run github:serokell/deploy-rs -- .#cryodev-main
 ```
 ### Rollback
 deploy-rs automatically rolls back if the new configuration fails health checks.
 Manual rollback:
 ```bash
 # List generations
 sudo nix-env -p /nix/var/nix/profiles/system --list-generations
 # Rollback to previous
 sudo nixos-rebuild switch --rollback
 ```
 ## Pull-based Deployment (cryodev-pi)
 ### How It Works
 1. Comin periodically polls the Git repository
 2. On changes, it builds and activates the new configuration
 3. Works through NAT without incoming connections
 ### Configuration
 ```nix
 # hosts/cryodev-pi/services/comin.nix
 {
  services.comin = {
    enable = true;
    remotes = [{
      name = "origin";
      url = "https://git.cryodev.xyz/steffen/cryodev-server.git";
      branches.main.name = "main";
    }];
  };
 }
 ```
 ### Monitoring
 Check Comin status:
 ```bash
 sudo systemctl status comin
 sudo journalctl -u comin -f
 ```
 Force immediate update:
 ```bash
 sudo systemctl restart comin
 ```
 ### Troubleshooting
 If Comin fails to build:
 ```bash
 # Check logs
 sudo journalctl -u comin --since "1 hour ago"
 # Manual build test
 cd /var/lib/comin/repo
 nix build .#nixosConfigurations.cryodev-pi.config.system.build.toplevel
 ```
 ## Manual Deployment
 For hosts not using automated deployment:
 ```bash
 # Build locally
 nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel
 # Deploy with nixos-rebuild
 nixos-rebuild switch --flake .#<hostname> --target-host root@<hostname>
 # Or using deploy-rs
 nix run github:serokell/deploy-rs -- .#<hostname>
 ```
 ## Testing Changes
 Before pushing, always verify:
 ```bash
 # Check flake validity
 nix flake check
 # Build configuration (dry-run)
 nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel --dry-run
 # Full build
 nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel
 ```
--- a/docs/deployment/dns.md
+++ b/docs/deployment/dns.md
@ -0,0 +1,93 @@
 # DNS Configuration
 Required DNS records for the cryodev infrastructure.
 ## Primary Domain (cryodev.xyz)
 ### A/AAAA Records
 | Hostname | Type | Value | Purpose |
 |----------|------|-------|---------|
 | `@` | A | `<SERVER_IP>` | Main server |
 | `@` | AAAA | `<SERVER_IPV6>` | Main server (IPv6) |
 | `mail` | A | `<SERVER_IP>` | Mail server |
 | `mail` | AAAA | `<SERVER_IPV6>` | Mail server (IPv6) |
 ### CNAME Records
 | Hostname | Type | Value | Purpose |
 |----------|------|-------|---------|
 | `git` | CNAME | `@` | Forgejo |
 | `headscale` | CNAME | `@` | Headscale |
 | `headplane` | CNAME | `@` | Headplane |
 | `netdata` | CNAME | `@` | Netdata Monitoring |
 ### Mail Records
 | Hostname | Type | Value | Purpose |
 |----------|------|-------|---------|
 | `@` | MX | `10 mail.cryodev.xyz.` | Mail delivery |
 | `@` | TXT | `"v=spf1 mx ~all"` | SPF |
 | `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC |
 | `mail._domainkey` | TXT | `"v=DKIM1; k=rsa; p=..."` | DKIM |
 ## Getting the DKIM Key
 After deploying the mailserver, retrieve the DKIM public key:
 ```bash
 sudo cat /var/dkim/cryodev.xyz.mail.txt
 ```
 Add this as a TXT record for `mail._domainkey.cryodev.xyz`.
 ## Verification
 ### Check DNS Propagation
 ```bash
 # A record
 dig A cryodev.xyz
 # MX record
 dig MX cryodev.xyz
 # SPF
 dig TXT cryodev.xyz
 # DKIM
 dig TXT mail._domainkey.cryodev.xyz
 # DMARC
 dig TXT _dmarc.cryodev.xyz
 ```
 ### Online Tools
 - [MXToolbox](https://mxtoolbox.com/) - Comprehensive DNS/mail testing
 - [Mail-tester](https://www.mail-tester.com/) - Email deliverability testing
 - [DMARC Analyzer](https://dmarcanalyzer.com/) - DMARC record validation
 ## TTL Recommendations
 For initial setup, use low TTLs (300 seconds) to allow quick changes.
 After verification, increase to:
 - A/AAAA records: 3600 (1 hour)
 - CNAME records: 3600 (1 hour)  
 - MX records: 3600 (1 hour)
 - TXT records: 3600 (1 hour)
 ## Firewall Requirements
 Ensure these ports are open on `cryodev-main`:
 | Port | Protocol | Service |
 |------|----------|---------|
 | 22 | TCP | SSH |
 | 80 | TCP | HTTP (ACME/redirect) |
 | 443 | TCP | HTTPS |
 | 25 | TCP | SMTP |
 | 465 | TCP | SMTPS |
 | 587 | TCP | SMTP Submission |
 | 993 | TCP | IMAPS |
--- a/docs/getting-started/first-install.md
+++ b/docs/getting-started/first-install.md
@ -0,0 +1,179 @@
 # Erstinstallation (x86_64 Server)
 Diese Anleitung beschreibt die **manuelle Installation** eines neuen x86_64 Servers (z.B. cryodev-main).
 > **Für Raspberry Pi:** Siehe [Neuen Raspberry Pi hinzufügen](new-client.md) - dort wird ein SD-Image automatisch gebaut.
 ## Übersicht
 Bei der Erstinstallation gibt es ein Henne-Ei-Problem:
 - SOPS-Secrets werden mit dem SSH-Host-Key verschlüsselt
 - Der SSH-Host-Key wird erst bei der Installation generiert
 - Daher: Erst installieren, dann Secrets konfigurieren
 ## Voraussetzungen
 - Bootbares NixOS ISO ([Minimal ISO](https://nixos.org/download/#nixos-iso))
 - Netzwerkverbindung
 - Host-Konfiguration in `hosts/<hostname>/` (ohne secrets.yaml)
 ## Schritt 1: Host-Konfiguration vorbereiten
 ### 1.1 Template kopieren
 ```bash
 cp -r templates/generic-server hosts/neuer-server
 ```
 ### 1.2 Hostname setzen
 `hosts/neuer-server/networking.nix`:
 ```nix
 {
  networking.hostName = "neuer-server";
 }
 ```
 ### 1.3 In flake.nix registrieren
 ```nix
 nixosConfigurations = {
  neuer-server = mkNixosConfiguration "x86_64-linux" [ ./hosts/neuer-server ];
 };
 ```
 ### 1.4 Placeholder secrets.yaml erstellen
 ```bash
 touch hosts/neuer-server/secrets.yaml
 ```
 ### 1.5 SOPS-Secrets temporär deaktivieren
 In `hosts/neuer-server/default.nix` alle `sops.secrets.*` Referenzen auskommentieren oder mit `lib.mkIf false` umgeben, bis die echten Secrets existieren.
 ## Schritt 2: Zielmaschine vorbereiten
 ### 2.1 NixOS ISO booten
 Von USB/CD booten.
 ### 2.2 Root-Passwort setzen (für SSH)
 ```bash
 passwd
 ```
 ### 2.3 IP-Adresse ermitteln
 ```bash
 ip a
 ```
 ### 2.4 Per SSH verbinden (optional)
 ```bash
 ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no nixos@<IP>
 sudo -i
 ```
 ## Schritt 3: Installation durchführen
 ### 3.1 Repository klonen
 ```bash
 nix-shell -p git
 git clone <GIT_REPO_URL> /tmp/nixos
 cd /tmp/nixos
 ```
 ### 3.2 Disk-Konfiguration anpassen
 **Wichtig:** Die Disk-ID muss zur Hardware passen!
 ```bash
 # Verfügbare Disks anzeigen
 lsblk -o NAME,SIZE,MODEL,SERIAL
 ls -la /dev/disk/by-id/
 ```
 In `hosts/neuer-server/disks.sh` oder `disks.nix` die richtige Disk-ID eintragen.
 ### 3.3 Install-Script ausführen
 ```bash
 bash scripts/install.sh -n neuer-server
 ```
 Das Script:
 1. Partitioniert die Disk (via disko oder disks.sh)
 2. Generiert hardware.nix (falls nicht vorhanden)
 3. Installiert NixOS
 ### 3.4 Reboot
 ```bash
 umount -Rl /mnt
 reboot
 ```
 ## Schritt 4: Nach dem ersten Boot
 ### 4.1 Einloggen
 Standard-Passwort: `changeme`
 ```bash
 passwd  # Sofort ändern!
 ```
 ### 4.2 SSH-Host-Key zu Age-Key konvertieren
 ```bash
 nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
 ```
 **Ausgabe notieren!** (z.B. `age1abc123...`)
 ### 4.3 Auf Entwicklungsrechner: SOPS konfigurieren
 `.sops.yaml` bearbeiten:
 ```yaml
 keys:
  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
  - &neuer_server_key age1abc123...  # Key von oben
 creation_rules:
  - path_regex: hosts/neuer-server/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *neuer_server_key
 ```
 ### 4.4 Secrets erstellen
 ```bash
 sops hosts/neuer-server/secrets.yaml
 ```
 Mindestens den Tailscale Auth-Key eintragen (siehe nächster Schritt).
 ### 4.5 SOPS-Referenzen wieder aktivieren
 Die in Schritt 1.5 auskommentierten `sops.secrets.*` Referenzen wieder aktivieren.
 ### 4.6 Konfiguration deployen
 ```bash
 # Lokal bauen und per SSH deployen
 nixos-rebuild switch --flake .#neuer-server --target-host root@<IP>
 ```
 ## Nächste Schritte
 - [Tailscale einrichten](../services/tailscale.md) - VPN-Verbindung
 - [Netdata konfigurieren](../services/netdata.md) - Monitoring
 - [CD einrichten](../deployment/cd.md) - Automatisches Deployment
--- a/docs/getting-started/new-client.md
+++ b/docs/getting-started/new-client.md
@ -0,0 +1,282 @@
 # Neuen Raspberry Pi Client hinzufügen
 Diese Anleitung beschreibt das Hinzufügen eines **neuen Raspberry Pi Clients** zur Infrastruktur.
 ## Übersicht: Der Ablauf
 ```
 1. Konfiguration erstellen     ──► Template kopieren, anpassen
         │
         ▼
 2. Zur Image-Pipeline hinzufügen ──► Workflow-Matrix erweitern
         │
         ▼
 3. Push auf main              ──► Forgejo baut automatisch SD-Image
         │
         ▼
 4. Image flashen & booten     ──► SD-Karte beschreiben, Pi starten
         │
         ▼
 5. SOPS konfigurieren         ──► Age-Key holen, Secrets erstellen
         │
         ▼
 6. Finales Deployment         ──► Tailscale etc. aktivieren
 ```
 ## Voraussetzungen
 - SSH-Zugang zu cryodev-main (für Tailscale Auth-Key)
 - Entwicklungsrechner mit Repository-Zugriff
 - SD-Karte (mindestens 8 GB)
 ---
 ## Schritt 1: Tailscale Auth-Key generieren
 **Auf cryodev-main** (per SSH):
 ```bash
 sudo headscale preauthkeys create --expiration 99y --reusable --user default
 ```
 **Ausgabe notieren!** (z.B. `tskey-preauth-abc123...`)
 ---
 ## Schritt 2: Host-Konfiguration erstellen
 ### 2.1 Template kopieren
 ```bash
 cp -r templates/raspberry-pi hosts/neuer-pi
 ```
 ### 2.2 Hostname setzen
 `hosts/neuer-pi/networking.nix`:
 ```nix
 {
  networking.hostName = "neuer-pi";
 }
 ```
 ### 2.3 In flake.nix registrieren
 ```nix
 nixosConfigurations = {
  # ... bestehende Hosts ...
  neuer-pi = mkNixosConfiguration "aarch64-linux" [ ./hosts/neuer-pi ];
 };
 ```
 ### 2.4 In constants.nix eintragen
 ```nix
 {
  hosts = {
    # ... bestehende Hosts ...
    neuer-pi = {
      ip = "100.64.0.X";  # Wird von Headscale vergeben
    };
  };
 }
 ```
 ### 2.5 Placeholder secrets.yaml erstellen
 ```bash
 touch hosts/neuer-pi/secrets.yaml
 ```
 ### 2.6 SOPS temporär deaktivieren
 In `hosts/neuer-pi/default.nix` die `sops.secrets.*` Referenzen auskommentieren, damit das Image ohne Secrets gebaut werden kann.
 ---
 ## Schritt 3: Zur Image-Pipeline hinzufügen
 Bearbeite `.forgejo/workflows/build-pi-image.yml`:
 ```yaml
 jobs:
  build-pi-images:
    strategy:
      matrix:
        # Neuen Host hier hinzufügen:
        host: [cryodev-pi, neuer-pi]
 ```
 ---
 ## Schritt 4: Push und Image bauen lassen
 ```bash
 git add .
 git commit -m "Add neuer-pi host configuration"
 git push
 ```
 Der Forgejo Workflow baut jetzt automatisch ein SD-Image für `neuer-pi`.
 **Warten** bis der Workflow fertig ist (30-60 Minuten). Status prüfen unter:
 `https://git.cryodev.xyz/steffen/cryodev-server/actions`
 ---
 ## Schritt 5: Image flashen
 ### 5.1 Image herunterladen
 Nach erfolgreichem Build unter **Releases**:
 ```bash
 wget https://git.cryodev.xyz/steffen/cryodev-server/releases/latest/download/neuer-pi-sd-image.img.zst
 ```
 ### 5.2 Dekomprimieren
 ```bash
 zstd -d neuer-pi-sd-image.img.zst -o neuer-pi.img
 ```
 ### 5.3 Auf SD-Karte schreiben
 **Achtung:** `/dev/sdX` durch das richtige Gerät ersetzen!
 ```bash
 lsblk  # Richtiges Gerät finden
 sudo dd if=neuer-pi.img of=/dev/sdX bs=4M conv=fsync status=progress
 ```
 ### 5.4 Booten
 1. SD-Karte in den Raspberry Pi einlegen
 2. Ethernet anschließen
 3. Strom anschließen
 4. Warten bis gebootet (ca. 2 Minuten)
 ---
 ## Schritt 6: SOPS konfigurieren
 ### 6.1 IP-Adresse finden
 Der Pi sollte per DHCP eine IP bekommen. Prüfe deinen Router oder scanne das Netzwerk:
 ```bash
 nmap -sn 192.168.1.0/24 | grep -B2 "Raspberry"
 ```
 ### 6.2 SSH verbinden
 ```bash
 ssh steffen@<IP>  # oder der konfigurierte User
 ```
 Standard-Passwort siehe `hosts/neuer-pi/users.nix`.
 ### 6.3 Age-Key ermitteln
 Auf dem Pi:
 ```bash
 nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
 ```
 **Ausgabe notieren!** (z.B. `age1xyz...`)
 ### 6.4 .sops.yaml aktualisieren
 Auf dem Entwicklungsrechner:
 ```yaml
 keys:
  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
  - &neuer_pi_key age1xyz...  # Der neue Key
 creation_rules:
  # ... bestehende Regeln ...
  - path_regex: hosts/neuer-pi/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *neuer_pi_key
 ```
 ### 6.5 Secrets erstellen
 ```bash
 sops hosts/neuer-pi/secrets.yaml
 ```
 Inhalt:
 ```yaml
 tailscale:
  auth-key: "tskey-preauth-abc123..."  # Key aus Schritt 1
 netdata:
  stream:
    child-uuid: "..."  # uuidgen
 ```
 ### 6.6 SOPS-Referenzen aktivieren
 Die in Schritt 2.6 auskommentierten `sops.secrets.*` Referenzen wieder aktivieren.
 ---
 ## Schritt 7: Finales Deployment
 ```bash
 git add .
 git commit -m "Configure SOPS secrets for neuer-pi"
 git push
 ```
 Da Comin auf dem Pi läuft, wird er die neue Konfiguration automatisch pullen.
 Alternativ manuell:
 ```bash
 nixos-rebuild switch --flake .#neuer-pi --target-host root@<IP>
 ```
 ---
 ## Schritt 8: Verifizieren
 ### Tailscale-Verbindung
 ```bash
 # Auf dem Pi
 tailscale status
 # Auf cryodev-main
 sudo headscale nodes list
 ```
 ### Netdata-Streaming
 Prüfe ob der neue Client im Netdata-Dashboard erscheint:
 `https://netdata.cryodev.xyz`
 ---
 ## Checkliste
 - [ ] Tailscale Auth-Key auf cryodev-main generiert
 - [ ] Host-Konfiguration erstellt (Template, flake.nix, constants.nix)
 - [ ] Host zur Workflow-Matrix hinzugefügt
 - [ ] Gepusht und auf Image-Build gewartet
 - [ ] SD-Karte geflasht und Pi gebootet
 - [ ] Age-Key ermittelt und in .sops.yaml eingetragen
 - [ ] secrets.yaml erstellt (Tailscale-Key, Netdata-UUID)
 - [ ] SOPS-Referenzen aktiviert und deployed
 - [ ] Tailscale-Verbindung funktioniert
 - [ ] Netdata-Streaming funktioniert
--- a/docs/getting-started/prerequisites.md
+++ b/docs/getting-started/prerequisites.md
@ -0,0 +1,63 @@
 # Prerequisites
 ## Required Tools
 Ensure you have the following tools installed on your local machine:
 | Tool | Purpose |
 |------|---------|
 | `nix` | Package manager with flakes enabled |
 | `sops` | Secret encryption/decryption |
 | `age` | Encryption backend for sops |
 | `ssh` | Remote access |
 ### Installing Nix
 Follow the [official Nix installation guide](https://nixos.org/download/).
 Enable flakes by adding to `~/.config/nix/nix.conf`:
 ```
 experimental-features = nix-command flakes
 ```
 ### Installing Other Tools
 With Nix:
 ```bash
 nix-shell -p sops age
 ```
 Or install globally via home-manager or system configuration.
 ## Repository Access
 Clone the repository:
 ```bash
 git clone https://git.cryodev.xyz/steffen/cryodev-server.git
 cd cryodev-server
 ```
 ## Development Shell
 Enter the development shell with all required tools:
 ```bash
 nix develop
 ```
 ## Verifying Setup
 Check that the flake is valid:
 ```bash
 nix flake check
 ```
 Build a host configuration (dry run):
 ```bash
 nix build .#nixosConfigurations.cryodev-main.config.system.build.toplevel --dry-run
 ```
--- a/docs/getting-started/reinstall.md
+++ b/docs/getting-started/reinstall.md
@ -0,0 +1,183 @@
 # Neuinstallation (Reinstall)
 Diese Anleitung beschreibt die **Neuinstallation** eines bestehenden Hosts, z.B. nach Hardwarewechsel oder bei Problemen.
 ## Unterschied zur Erstinstallation
 | Aspekt | Erstinstallation | Neuinstallation |
 |--------|------------------|-----------------|
 | SOPS-Secrets | Noch nicht vorhanden | Bereits konfiguriert |
 | SSH-Host-Key | Neu generiert | **Muss wiederhergestellt werden!** |
 | Disk-IDs | Neu ermitteln | Oft geändert (neue Hardware) |
 | secrets.yaml | Wird erstellt | Bereits vorhanden |
 ## Wichtig: SSH-Host-Key Problem
 Bei einer Neuinstallation wird ein **neuer SSH-Host-Key** generiert. Dieser stimmt nicht mehr mit dem Age-Key in `.sops.yaml` überein!
 ### Lösungsmöglichkeiten
 **Option A: Alten Host-Key sichern und wiederherstellen** (empfohlen)
 **Option B: Neuen Key generieren und SOPS aktualisieren**
 ## Voraussetzungen
 - Backup des alten SSH-Host-Keys (falls Option A)
 - Zugriff auf `.sops.yaml` und die Admin-Age-Keys
 - Bootbares NixOS ISO
 ## Schritt 1: Vorbereitung (vor der Installation)
 ### 1.1 Alten SSH-Host-Key sichern (Option A)
 Falls der alte Host noch läuft:
 ```bash
 # Auf dem alten Host
 sudo cat /etc/ssh/ssh_host_ed25519_key > ~/ssh_host_ed25519_key.backup
 sudo cat /etc/ssh/ssh_host_ed25519_key.pub > ~/ssh_host_ed25519_key.pub.backup
 ```
 Dateien sicher auf den Entwicklungsrechner kopieren.
 ### 1.2 Disk-IDs ermitteln
 **Bei neuer Hardware** ändern sich die Disk-IDs!
 ```bash
 # Im NixOS Live-System
 lsblk -o NAME,SIZE,MODEL,SERIAL
 ls -la /dev/disk/by-id/
 ```
 Die neue Disk-ID in `hosts/<hostname>/disks.sh` oder `disks.nix` eintragen:
 ```bash
 # Beispiel disks.sh
 DISK="/dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_XXXXX"
 ```
 ## Schritt 2: Installation durchführen
 ### 2.1 NixOS ISO booten
 Von USB/CD booten, Root-Passwort setzen, per SSH verbinden.
 ### 2.2 Repository klonen
 ```bash
 sudo -i
 nix-shell -p git
 git clone <GIT_REPO_URL> /tmp/nixos
 cd /tmp/nixos
 ```
 ### 2.3 Disk-Konfiguration prüfen
 ```bash
 # Aktuelle Disk-IDs anzeigen
 ls -la /dev/disk/by-id/
 # Mit Konfiguration vergleichen
 cat hosts/<hostname>/disks.sh | grep DISK
 ```
 **Falls nötig:** Disk-ID in der Konfiguration anpassen.
 ### 2.4 Install-Script ausführen
 ```bash
 bash scripts/install.sh -n <hostname>
 ```
 ### 2.5 SSH-Host-Key wiederherstellen (Option A)
 **Vor dem Reboot!**
 ```bash
 # Host-Key vom Backup wiederherstellen
 cp /path/to/ssh_host_ed25519_key.backup /mnt/etc/ssh/ssh_host_ed25519_key
 cp /path/to/ssh_host_ed25519_key.pub.backup /mnt/etc/ssh/ssh_host_ed25519_key.pub
 chmod 600 /mnt/etc/ssh/ssh_host_ed25519_key
 chmod 644 /mnt/etc/ssh/ssh_host_ed25519_key.pub
 ```
 ### 2.6 Reboot
 ```bash
 umount -Rl /mnt
 reboot
 ```
 ## Schritt 3: Nach dem Reboot
 ### Bei Option A (Key wiederhergestellt)
 SOPS-Secrets sollten automatisch funktionieren. Testen:
 ```bash
 sudo cat /run/secrets/tailscale/auth-key
 ```
 ### Bei Option B (Neuer Key)
 Der Host kann die Secrets nicht entschlüsseln. Neuen Key konfigurieren:
 ```bash
 # Neuen Age-Key ermitteln
 nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
 ```
 Auf dem Entwicklungsrechner:
 ```bash
 # .sops.yaml aktualisieren mit neuem Key
 vim .sops.yaml
 # Secrets mit neuem Key neu verschlüsseln
 sops updatekeys hosts/<hostname>/secrets.yaml
 ```
 Dann Konfiguration neu deployen:
 ```bash
 nixos-rebuild switch --flake .#<hostname> --target-host root@<IP>
 ```
 ## Häufige Probleme
 ### "No secret key available"
 SOPS kann die Secrets nicht entschlüsseln. Ursache:
 - SSH-Host-Key stimmt nicht mit Age-Key in `.sops.yaml` überein
 Lösung: Option B durchführen (neuen Key konfigurieren).
 ### "Device not found" beim Partitionieren
 Disk-ID in `disks.sh`/`disks.nix` ist falsch.
 ```bash
 # Richtige ID finden
 ls -la /dev/disk/by-id/
 ```
 ### Hardware-Config veraltet
 Bei neuer Hardware muss `hardware.nix` neu generiert werden:
 ```bash
 # Install-Script generiert automatisch neu, falls Datei fehlt
 rm hosts/<hostname>/hardware.nix
 bash scripts/install.sh -n <hostname>
 ```
 ## Checkliste
 - [ ] Alten SSH-Host-Key gesichert (falls möglich)
 - [ ] Disk-IDs in Konfiguration geprüft/aktualisiert
 - [ ] Installation durchgeführt
 - [ ] SSH-Host-Key wiederhergestellt ODER neuen Key in SOPS konfiguriert
 - [ ] Secrets funktionieren (`sudo cat /run/secrets/...`)
 - [ ] Tailscale verbunden (`tailscale status`)
--- a/docs/getting-started/sd-image.md
+++ b/docs/getting-started/sd-image.md
@ -0,0 +1,116 @@
 # SD-Karten-Images für Raspberry Pi
 Das Repository baut automatisch SD-Karten-Images für alle konfigurierten Raspberry Pi Hosts.
 ## Automatischer Build
 Bei Änderungen an `main` werden automatisch Images für alle Pi-Hosts gebaut und als Release veröffentlicht.
 **Download:** [Releases auf Forgejo](https://git.cryodev.xyz/steffen/cryodev-server/releases)
 ## Verfügbare Images
 | Host | Image-Name |
 |------|------------|
 | `cryodev-pi` | `cryodev-pi-sd-image.img.zst` |
 Neue Hosts werden automatisch gebaut, wenn sie zur Workflow-Matrix hinzugefügt werden.
 ## Image flashen
 ### 1. Herunterladen
 ```bash
 wget https://git.cryodev.xyz/.../releases/latest/download/<hostname>-sd-image.img.zst
 wget https://git.cryodev.xyz/.../releases/latest/download/<hostname>-sd-image.img.zst.sha256
 # Checksum prüfen
 sha256sum -c <hostname>-sd-image.img.zst.sha256
 ```
 ### 2. Dekomprimieren
 ```bash
 zstd -d <hostname>-sd-image.img.zst -o <hostname>.img
 ```
 ### 3. Auf SD-Karte schreiben
 ```bash
 # Richtiges Gerät finden
 lsblk
 # Schreiben (ACHTUNG: richtiges Gerät wählen!)
 sudo dd if=<hostname>.img of=/dev/sdX bs=4M conv=fsync status=progress
 ```
 Alternativ: `balenaEtcher` oder `Raspberry Pi Imager` verwenden.
 ## Was ist im Image?
 - Vollständige NixOS-Installation für den spezifischen Host
 - Alle konfigurierten Services (außer Secrets)
 - SSH-Server aktiviert
 - Automatische Root-Partition-Erweiterung beim ersten Boot
 - Comin für automatische Updates
 ## Was fehlt?
 **SOPS-Secrets** können nicht im Image enthalten sein (Henne-Ei-Problem mit SSH-Host-Key).
 Nach dem ersten Boot:
 1. Age-Key vom Pi holen
 2. `.sops.yaml` aktualisieren
 3. `secrets.yaml` erstellen
 4. Konfiguration deployen
 Siehe [Neuen Client hinzufügen](new-client.md) für die vollständige Anleitung.
 ## Neuen Host zur Pipeline hinzufügen
 1. Host-Konfiguration in `hosts/<hostname>/` erstellen
 2. In `.forgejo/workflows/build-pi-image.yml` zur Matrix hinzufügen:
 ```yaml
 matrix:
  host: [cryodev-pi, neuer-host]  # <- hier hinzufügen
 ```
 3. Push auf `main` → Image wird automatisch gebaut
 ## Manuell bauen
 ```bash
 # Auf aarch64 (z.B. anderem Pi)
 nix build .#nixosConfigurations.<hostname>.config.system.build.sdImage
 # Auf x86_64 mit QEMU-Emulation (langsam)
 nix build .#nixosConfigurations.<hostname>.config.system.build.sdImage \
  --extra-platforms aarch64-linux
 ```
 Voraussetzung auf x86_64:
 ```nix
 {
  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 }
 ```
 ## Troubleshooting
 ### Workflow schlägt fehl
 - Prüfe ob `sd-image.nix` in der Host-Konfiguration importiert wird
 - Prüfe ob binfmt auf cryodev-main aktiviert ist
 ### Image bootet nicht
 - SD-Karte korrekt beschrieben?
 - Andere SD-Karte versuchen
 - Stromversorgung prüfen (min. 3A für Pi 4)
 ### Kein Netzwerk
 - Ethernet-Kabel prüfen
 - DHCP-Server im Netzwerk?
--- a/docs/index.md
+++ b/docs/index.md
@ -0,0 +1,94 @@
 # Cryodev NixOS Configuration Documentation
 Willkommen zur Dokumentation der **cryodev** NixOS-Infrastruktur.
 ## Quick Links
 ### Getting Started
 - [Voraussetzungen](getting-started/prerequisites.md) - Benötigte Tools
 - [Neuen Raspberry Pi hinzufügen](getting-started/new-client.md) - Kompletter Workflow für neue Clients
 - [SD-Image Referenz](getting-started/sd-image.md) - Details zum Image-Build
 - [Erstinstallation (Server)](getting-started/first-install.md) - Bootstrap für x86_64 Hosts
 - [Neuinstallation](getting-started/reinstall.md) - Reinstall mit Hardware-Änderungen
 ### Services
 - [SOPS Secrets](services/sops.md) - Geheimnisverwaltung mit sops-nix
 - [Headscale](services/headscale.md) - Self-hosted Tailscale Server
 - [Headplane](services/headplane.md) - Web-UI für Headscale
 - [Tailscale](services/tailscale.md) - Mesh-VPN Client
 - [Mailserver](services/mailserver.md) - E-Mail Stack (Postfix/Dovecot)
 - [Forgejo](services/forgejo.md) - Git-Hosting mit CI/CD
 - [Netdata](services/netdata.md) - Monitoring und Alerting
 ### Deployment
 - [Continuous Deployment](deployment/cd.md) - Push- und Pull-basiertes Deployment
 - [DNS-Konfiguration](deployment/dns.md) - Benötigte DNS-Einträge
 ## Architektur
 ```
                    Internet
                        |
                   cryodev.xyz
                        |
              +-------------------+
              |   cryodev-main    |
              |   (x86_64 Server) |
              +-------------------+
              | - Headscale       |
              | - Headplane       |
              | - Forgejo         |
              | - Mailserver      |
              | - Netdata Parent  |
              +-------------------+
                        |
               Tailscale Mesh VPN
                        |
              +-------------------+
              |   cryodev-pi      |
              | (Raspberry Pi 4)  |
              +-------------------+
              | - Tailscale       |
              | - Netdata Child   |
              | - Comin (GitOps)  |
              +-------------------+
 ```
 ## Installations-Szenarien
 | Szenario | Beschreibung | Anleitung |
 |----------|--------------|-----------|
 | **Neuer Raspberry Pi** | Config erstellen → Image bauen → Flashen | [new-client.md](getting-started/new-client.md) |
 | **Erstinstallation (Server)** | x86_64 Host, manuelle Installation | [first-install.md](getting-started/first-install.md) |
 | **Neuinstallation** | Bestehender Host, neue Hardware | [reinstall.md](getting-started/reinstall.md) |
 Für Raspberry Pi: [SD-Image Referenz](getting-started/sd-image.md)
 ## Verzeichnisstruktur
 ```
 .
 ├── flake.nix           # Entry point, inputs and outputs
 ├── constants.nix       # Zentrale Config (Domains, IPs, Ports)
 ├── hosts/              # Host-spezifische Konfigurationen
 │   ├── cryodev-main/
 │   └── cryodev-pi/
 ├── modules/            # Wiederverwendbare NixOS-Module
 │   └── nixos/
 ├── pkgs/               # Eigene Pakete
 ├── overlays/           # Nixpkgs Overlays
 ├── templates/          # Templates für neue Hosts
 ├── scripts/            # Helper-Scripts (install.sh)
 ├── apps/               # Nix Apps (rebuild)
 └── lib/                # Helper-Funktionen (utils.nix)
 ```
 ## Deployment-Strategien
 | Host | Strategie | Tool | Beschreibung |
 |------|-----------|------|--------------|
 | `cryodev-main` | Push-basiert | deploy-rs via Forgejo Actions | Sofortige Updates bei Push |
 | `cryodev-pi` | Pull-basiert | Comin | Pollt Repository auf Änderungen |
--- a/docs/services/forgejo.md
+++ b/docs/services/forgejo.md
@ -0,0 +1,149 @@
 # Forgejo
 Forgejo is a self-hosted Git service (fork of Gitea) with built-in CI/CD Actions.
 ## References
 - [Forgejo Documentation](https://forgejo.org/docs/)
 - [Forgejo Actions](https://forgejo.org/docs/latest/user/actions/)
 ## Setup
 ### DNS
 Set a CNAME record for `git.cryodev.xyz` pointing to your main domain.
 ### Configuration
 ```nix
 # hosts/cryodev-main/services/forgejo.nix
 { config, ... }:
 {
  services.forgejo = {
    enable = true;
    settings = {
      server = {
        DOMAIN = "git.cryodev.xyz";
        ROOT_URL = "https://git.cryodev.xyz";
      };
      mailer = {
        ENABLED = true;
        FROM = "forgejo@cryodev.xyz";
      };
    };
  };
 }
 ```
 ## Forgejo Runner
 The runner executes CI/CD pipelines defined in `.forgejo/workflows/`.
 ### Get Runner Token
 1. Go to Forgejo Admin Panel
 2. Navigate to Actions > Runners
 3. Create a new runner and copy the token
 ### Add to Secrets
 ```bash
 sops hosts/cryodev-main/secrets.yaml
 ```
 ```yaml
 forgejo-runner:
  token: "your-runner-token"
 ```
 ### Configuration
 ```nix
 {
  sops.secrets."forgejo-runner/token" = { };
  services.gitea-actions-runner = {
    instances.default = {
      enable = true;
      url = "https://git.cryodev.xyz";
      tokenFile = config.sops.secrets."forgejo-runner/token".path;
      labels = [ "ubuntu-latest:docker://node:20" ];
    };
  };
 }
 ```
 ## CI/CD Workflows
 ### deploy-rs Workflow
 `.forgejo/workflows/deploy.yaml`:
 ```yaml
 name: Deploy
 on:
  push:
    branches: [main]
 jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Nix
        uses: cachix/install-nix-action@v24
      - name: Deploy
        env:
          SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
        run: |
          mkdir -p ~/.ssh
          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
          chmod 600 ~/.ssh/id_ed25519
          nix run .#deploy
 ```
 ## Administration
 ### Create Admin User
 ```bash
 sudo -u forgejo forgejo admin user create \
  --username admin \
  --password changeme \
  --email admin@cryodev.xyz \
  --admin
 ```
 ### Reset User Password
 ```bash
 sudo -u forgejo forgejo admin user change-password \
  --username USER \
  --password NEWPASS
 ```
 ## Troubleshooting
 ### Check Service Status
 ```bash
 sudo systemctl status forgejo
 sudo systemctl status gitea-runner-default
 ```
 ### View Logs
 ```bash
 sudo journalctl -u forgejo -f
 sudo journalctl -u gitea-runner-default -f
 ```
 ### Database Issues
 Forgejo uses SQLite by default. Database location:
 ```bash
 ls -la /var/lib/forgejo/data/
 ```
--- a/docs/services/headplane.md
+++ b/docs/services/headplane.md
@ -0,0 +1,107 @@
 # Headplane
 Headplane is a web-based admin interface for Headscale.
 ## References
 - [GitHub](https://github.com/tale/headplane)
 ## Setup
 ### DNS
 Set a CNAME record for `headplane.cryodev.xyz` pointing to your main domain.
 ### Generate Secrets
 **Cookie Secret** (for session management):
 ```bash
 nix-shell -p openssl --run 'openssl rand -hex 16'
 ```
 **Agent Pre-Auth Key** (for Headplane's built-in agent):
 ```bash
 # First, create a dedicated user
 sudo headscale users create headplane-agent
 # Then create a reusable pre-auth key
 sudo headscale preauthkeys create --expiration 99y --reusable --user headplane-agent
 ```
 ### Add to Secrets
 Edit `hosts/cryodev-main/secrets.yaml`:
 ```bash
 sops hosts/cryodev-main/secrets.yaml
 ```
 ```yaml
 headplane:
  cookie_secret: "your-generated-hex-string"
  agent_pre_authkey: "your-preauth-key"
 ```
 ### Configuration
 ```nix
 # hosts/cryodev-main/services/headplane.nix
 { config, ... }:
 {
  sops.secrets."headplane/cookie_secret" = { };
  sops.secrets."headplane/agent_pre_authkey" = { };
  services.headplane = {
    enable = true;
    settings = {
      server = {
        cookie_secret_file = config.sops.secrets."headplane/cookie_secret".path;
      };
      headscale = {
        url = "https://headscale.cryodev.xyz";
      };
      agent = {
        enable = true;
        authkey_file = config.sops.secrets."headplane/agent_pre_authkey".path;
      };
    };
  };
 }
 ```
 ## Usage
 Access Headplane at `https://headplane.cryodev.xyz`.
 ### Features
 - View and manage users
 - View connected nodes
 - Manage routes and exit nodes
 - View pre-auth keys
 ## Troubleshooting
 ### Check Service Status
 ```bash
 sudo systemctl status headplane
 ```
 ### View Logs
 ```bash
 sudo journalctl -u headplane -f
 ```
 ### Agent Not Connecting
 Verify the agent pre-auth key is valid:
 ```bash
 sudo headscale preauthkeys list --user headplane-agent
 ```
 If expired, create a new one and update the secrets file.
--- a/docs/services/headscale.md
+++ b/docs/services/headscale.md
@ -0,0 +1,116 @@
 # Headscale
 Headscale is an open-source, self-hosted implementation of the Tailscale control server.
 ## References
 - [Website](https://headscale.net/stable/)
 - [GitHub](https://github.com/juanfont/headscale)
 - [Example configuration](https://github.com/juanfont/headscale/blob/main/config-example.yaml)
 ## Setup
 ### DNS
 Set a CNAME record for `headscale.cryodev.xyz` pointing to your main domain.
 ### Configuration
 ```nix
 # hosts/cryodev-main/services/headscale.nix
 {
  services.headscale = {
    enable = true;
    openFirewall = true;
  };
 }
 ```
 ## Usage
 ### Create a User
 ```bash
 sudo headscale users create <USERNAME>
 ```
 ### List Users
 ```bash
 sudo headscale users list
 ```
 ### Create Pre-Auth Key
 ```bash
 sudo headscale preauthkeys create --expiration 99y --reusable --user <USER_ID>
 ```
 The pre-auth key is used by clients to automatically authenticate and join the tailnet.
 ### List Nodes
 ```bash
 sudo headscale nodes list
 ```
 ### Delete a Node
 ```bash
 sudo headscale nodes delete -i <NODE_ID>
 ```
 ### Rename a Node
 ```bash
 sudo headscale nodes rename -i <NODE_ID> new-name
 ```
 ## ACL Configuration
 Access Control Lists define which nodes can communicate with each other.
 ### Validate ACL File
 ```bash
 sudo headscale policy check --file /path/to/acl.hujson
 ```
 ### Example ACL
 ```json
 {
  "acls": [
    {
      "action": "accept",
      "src": ["*"],
      "dst": ["*:*"]
    }
  ]
 }
 ```
 ## Troubleshooting
 ### Check Service Status
 ```bash
 sudo systemctl status headscale
 ```
 ### View Logs
 ```bash
 sudo journalctl -u headscale -f
 ```
 ### Test DERP Connectivity
 ```bash
 curl -I https://headscale.cryodev.xyz/derp
 ```
 ## Integration
 - [Headplane](headplane.md) - Web UI for managing Headscale
 - [Tailscale Client](tailscale.md) - Connect clients to Headscale
--- a/docs/services/mailserver.md
+++ b/docs/services/mailserver.md
@ -0,0 +1,147 @@
 # Mailserver
 NixOS mailserver module providing a complete email stack with Postfix and Dovecot.
 ## References
 - [Simple NixOS Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver)
 ## Setup
 ### DNS Records
 | Type | Hostname | Value |
 |------|----------|-------|
 | A | `mail` | `<SERVER_IP>` |
 | AAAA | `mail` | `<SERVER_IPV6>` |
 | MX | `@` | `10 mail.cryodev.xyz.` |
 | TXT | `@` | `"v=spf1 mx ~all"` |
 | TXT | `_dmarc` | `"v=DMARC1; p=none"` |
 DKIM records are generated automatically after first deployment.
 ### Generate Password Hashes
 ```bash
 nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'
 ```
 ### Add to Secrets
 ```bash
 sops hosts/cryodev-main/secrets.yaml
 ```
 ```yaml
 mailserver:
  accounts:
    admin: "$2y$05$..."
    forgejo: "$2y$05$..."
 ```
 ### Configuration
 ```nix
 # hosts/cryodev-main/services/mailserver.nix
 { config, ... }:
 {
  sops.secrets."mailserver/accounts/admin" = { };
  sops.secrets."mailserver/accounts/forgejo" = { };
  mailserver = {
    enable = true;
    fqdn = "mail.cryodev.xyz";
    domains = [ "cryodev.xyz" ];
    loginAccounts = {
      "admin@cryodev.xyz" = {
        hashedPasswordFile = config.sops.secrets."mailserver/accounts/admin".path;
      };
      "forgejo@cryodev.xyz" = {
        hashedPasswordFile = config.sops.secrets."mailserver/accounts/forgejo".path;
        sendOnly = true;
      };
    };
  };
 }
 ```
 ## DKIM Setup
 After first deployment, get the DKIM public key:
 ```bash
 sudo cat /var/dkim/cryodev.xyz.mail.txt
 ```
 Add this as a TXT record:
 | Type | Hostname | Value |
 |------|----------|-------|
 | TXT | `mail._domainkey` | `v=DKIM1; k=rsa; p=...` |
 ## Testing
 ### Send Test Email
 ```bash
 echo "Test" | mail -s "Test Subject" recipient@example.com
 ```
 ### Check Mail Queue
 ```bash
 sudo postqueue -p
 ```
 ### View Logs
 ```bash
 sudo journalctl -u postfix -f
 sudo journalctl -u dovecot2 -f
 ```
 ### Test SMTP
 ```bash
 openssl s_client -connect mail.cryodev.xyz:587 -starttls smtp
 ```
 ### Verify DNS Records
 - [MXToolbox](https://mxtoolbox.com/)
 - [Mail-tester](https://www.mail-tester.com/)
 ## Troubleshooting
 ### Emails Not Sending
 Check Postfix status:
 ```bash
 sudo systemctl status postfix
 ```
 Check firewall (ports 25, 465, 587 must be open):
 ```bash
 sudo iptables -L -n | grep -E '25|465|587'
 ```
 ### DKIM Failing
 Verify the DNS record matches the generated key:
 ```bash
 dig TXT mail._domainkey.cryodev.xyz
 ```
 ### SPF Failing
 Verify SPF record:
 ```bash
 dig TXT cryodev.xyz
 ```
 Should return: `"v=spf1 mx ~all"`
--- a/docs/services/netdata.md
+++ b/docs/services/netdata.md
@ -0,0 +1,181 @@
 # Netdata Monitoring
 Netdata provides real-time performance monitoring with parent/child streaming.
 ## Architecture
 ```
 ┌─────────────────┐     Stream over      ┌─────────────────┐
 │  cryodev-pi     │  ───────────────────>│  cryodev-main   │
 │  (Child Node)   │     Tailscale VPN    │  (Parent Node)  │
 └─────────────────┘                      └─────────────────┘
                                                  │
                                                  v
                                         https://netdata.cryodev.xyz
 ```
 ## References
 - [Netdata Documentation](https://learn.netdata.cloud/)
 - [Streaming Configuration](https://learn.netdata.cloud/docs/streaming/streaming-configuration-reference)
 ## Parent Node (cryodev-main)
 ### DNS
 Set a CNAME record for `netdata.cryodev.xyz` pointing to your main domain.
 ### Generate Stream API Key
 ```bash
 uuidgen
 ```
 ### Configuration
 ```nix
 # hosts/cryodev-main/services/netdata.nix
 { config, ... }:
 {
  sops.secrets."netdata/stream-api-key" = { };
  sops.templates."netdata-stream.conf" = {
    content = ''
      [${config.sops.placeholder."netdata/stream-api-key"}]
      enabled = yes
      default history = 3600
      default memory mode = ram
      health enabled by default = auto
      allow from = *
    '';
    owner = "netdata";
  };
  services.netdata = {
    enable = true;
    configDir."stream.conf" = config.sops.templates."netdata-stream.conf".path;
  };
 }
 ```
 ## Child Node (cryodev-pi)
 ### Generate Child UUID
 ```bash
 uuidgen
 ```
 ### Add to Secrets
 ```bash
 sops hosts/cryodev-pi/secrets.yaml
 ```
 ```yaml
 netdata:
  stream:
    child-uuid: "your-generated-uuid"
 ```
 Note: The stream API key must match the parent's key. You can either:
 1. Share the same secret between hosts (complex with SOPS)
 2. Hardcode a known API key in both configurations
 ### Configuration
 ```nix
 # hosts/cryodev-pi/services/netdata.nix
 { config, constants, ... }:
 {
  sops.secrets."netdata/stream/child-uuid" = { };
  sops.templates."netdata-stream.conf" = {
    content = ''
      [stream]
      enabled = yes
      destination = ${constants.hosts.cryodev-main.ip}:19999
      api key = YOUR_STREAM_API_KEY
      send charts matching = *
    '';
    owner = "netdata";
  };
  services.netdata = {
    enable = true;
    configDir."stream.conf" = config.sops.templates."netdata-stream.conf".path;
  };
 }
 ```
 ## Email Alerts
 Configure Netdata to send alerts via the mailserver:
 ```nix
 {
  services.netdata.configDir."health_alarm_notify.conf" = pkgs.writeText "notify.conf" ''
    SEND_EMAIL="YES"
    EMAIL_SENDER="netdata@cryodev.xyz"
    DEFAULT_RECIPIENT_EMAIL="admin@cryodev.xyz"
  '';
 }
 ```
 ## Usage
 ### Access Dashboard
 Open `https://netdata.cryodev.xyz` in your browser.
 ### View Child Nodes
 Child nodes appear in the left sidebar under "Nodes".
 ### Check Streaming Status
 On parent:
 ```bash
 curl -s http://localhost:19999/api/v1/info | jq '.hosts'
 ```
 On child:
 ```bash
 curl -s http://localhost:19999/api/v1/info | jq '.streaming'
 ```
 ## Troubleshooting
 ### Check Service Status
 ```bash
 sudo systemctl status netdata
 ```
 ### View Logs
 ```bash
 sudo journalctl -u netdata -f
 ```
 ### Child Not Streaming
 1. Verify network connectivity:
   ```bash
   tailscale ping cryodev-main
   nc -zv <parent-ip> 19999
   ```
 2. Check API key matches between parent and child
 3. Verify firewall allows port 19999 on parent
 ### High Memory Usage
 Adjust history settings in `netdata.conf`:
 ```ini
 [global]
 history = 1800  # seconds to retain
 memory mode = ram
 ```
--- a/docs/services/sops.md
+++ b/docs/services/sops.md
@ -0,0 +1,174 @@
 # SOPS Secret Management
 Atomic secret provisioning for NixOS using [sops-nix](https://github.com/Mic92/sops-nix).
 ## Overview
 Secrets are encrypted with `age` using SSH host keys, ensuring:
 - No plaintext secrets in the repository
 - Secrets are decrypted at activation time
 - Each host can only decrypt its own secrets
 ## Setup
 ### 1. Get Host's Age Public Key
 After a host is installed, extract its age key from the SSH host key:
 ```bash
 nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <HOST_IP> | ssh-to-age'
 ```
 Or locally on the host:
 ```bash
 nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
 ```
 ### 2. Configure .sops.yaml
 Add the host key to `.sops.yaml`:
 ```yaml
 keys:
  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
  - &main_key age1...  # cryodev-main
  - &pi_key age1...    # cryodev-pi
 creation_rules:
  - path_regex: hosts/cryodev-main/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *main_key
  - path_regex: hosts/cryodev-pi/secrets.yaml$
    key_groups:
      - age:
        - *admin_key
        - *pi_key
 ```
 ### 3. Create Secrets File
 ```bash
 sops hosts/<hostname>/secrets.yaml
 ```
 This opens your editor. Add secrets in YAML format:
 ```yaml
 tailscale:
  auth-key: "tskey-..."
 some-service:
  password: "secret123"
 ```
 ## Usage in Modules
 ### Declaring Secrets
 ```nix
 { config, ... }:
 {
  sops.secrets.my-secret = {
    # Optional: set owner/group
    owner = "myservice";
    group = "myservice";
  };
 }
 ```
 ### Using Secrets
 Reference the secret path in service configuration:
 ```nix
 {
  services.myservice = {
    passwordFile = config.sops.secrets.my-secret.path;
  };
 }
 ```
 ### Using Templates
 For secrets that need to be embedded in config files:
 ```nix
 {
  sops.secrets."netdata/stream-api-key" = { };
  sops.templates."netdata-stream.conf" = {
    content = ''
      [stream]
      enabled = yes
      api key = ${config.sops.placeholder."netdata/stream-api-key"}
    '';
    owner = "netdata";
  };
  services.netdata.configDir."stream.conf" = 
    config.sops.templates."netdata-stream.conf".path;
 }
 ```
 ## Common Secrets
 ### cryodev-main
 ```yaml
 mailserver:
  accounts:
    forgejo: "$2y$05$..."  # bcrypt hash
    admin: "$2y$05$..."
 forgejo-runner:
  token: "..."
 headplane:
  cookie_secret: "..."       # openssl rand -hex 16
  agent_pre_authkey: "..."   # headscale preauthkey
 tailscale:
  auth-key: "tskey-..."
 ```
 ### cryodev-pi
 ```yaml
 tailscale:
  auth-key: "tskey-..."
 netdata:
  stream:
    child-uuid: "..."  # uuidgen
 ```
 ## Generating Secret Values
 | Secret | Command |
 |--------|---------|
 | Mailserver password | `nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'` |
 | Random hex token | `nix-shell -p openssl --run 'openssl rand -hex 16'` |
 | UUID | `uuidgen` |
 | Tailscale preauth | `sudo headscale preauthkeys create --expiration 99y --reusable --user default` |
 ## Updating Keys
 After modifying `.sops.yaml`, update existing secrets files:
 ```bash
 sops --config .sops.yaml updatekeys hosts/<hostname>/secrets.yaml
 ```
 ## Troubleshooting
 ### "No matching keys found"
 Ensure the host's age key is in `.sops.yaml` and you've run `updatekeys`.
 ### Secret not decrypting on host
 Check that `/etc/ssh/ssh_host_ed25519_key` exists and matches the public key in `.sops.yaml`.
--- a/docs/services/tailscale.md
+++ b/docs/services/tailscale.md
@ -0,0 +1,117 @@
 # Tailscale Client
 Tailscale clients connect to the self-hosted Headscale server to join the mesh VPN.
 ## References
 - [Tailscale Documentation](https://tailscale.com/kb)
 - [Headscale Client Setup](https://headscale.net/running-headscale-linux/)
 ## Setup
 ### Generate Auth Key
 On the Headscale server (cryodev-main):
 ```bash
 sudo headscale preauthkeys create --expiration 99y --reusable --user default
 ```
 ### Add to Secrets
 ```bash
 sops hosts/<hostname>/secrets.yaml
 ```
 ```yaml
 tailscale:
  auth-key: "your-preauth-key"
 ```
 ### Configuration
 ```nix
 # In your host configuration
 { config, ... }:
 {
  sops.secrets."tailscale/auth-key" = { };
  services.tailscale = {
    enable = true;
    authKeyFile = config.sops.secrets."tailscale/auth-key".path;
    extraUpFlags = [
      "--login-server=https://headscale.cryodev.xyz"
    ];
  };
 }
 ```
 ## Usage
 ### Check Status
 ```bash
 tailscale status
 ```
 ### View IP Address
 ```bash
 tailscale ip
 ```
 ### Ping Another Node
 ```bash
 tailscale ping <hostname>
 ```
 ### SSH to Another Node
 ```bash
 ssh user@<hostname>
 # or using Tailscale IP
 ssh user@100.64.0.X
 ```
 ## MagicDNS
 With Headscale's MagicDNS enabled, you can reach nodes by hostname:
 ```bash
 ping cryodev-pi
 ssh steffen@cryodev-main
 ```
 ## Troubleshooting
 ### Check Service Status
 ```bash
 sudo systemctl status tailscaled
 ```
 ### View Logs
 ```bash
 sudo journalctl -u tailscaled -f
 ```
 ### Re-authenticate
 If the node is not connecting:
 ```bash
 sudo tailscale up --login-server=https://headscale.cryodev.xyz --force-reauth
 ```
 ### Node Not Appearing in Headscale
 Check the auth key is valid:
 ```bash
 # On Headscale server
 sudo headscale preauthkeys list --user default
 ```
 Verify the login server URL is correct in the client configuration.
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,627 @@
 {
  "nodes": {
    "blobs": {
      "flake": false,
      "locked": {
        "lastModified": 1604995301,
        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
        "owner": "simple-nixos-mailserver",
        "repo": "blobs",
        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
        "type": "gitlab"
      },
      "original": {
        "owner": "simple-nixos-mailserver",
        "repo": "blobs",
        "type": "gitlab"
      }
    },
    "comin": {
      "inputs": {
        "flake-compat": "flake-compat",
        "nixpkgs": [
          "nixpkgs"
        ],
        "treefmt-nix": "treefmt-nix"
      },
      "locked": {
        "lastModified": 1772962094,
        "narHash": "sha256-9+/PHrDNDUy9iiN7seOhcxq3KoVlCAmCim6HXuKTI24=",
        "owner": "nlewo",
        "repo": "comin",
        "rev": "269ef4334f202b226eef804c0be0201891fb9c5d",
        "type": "github"
      },
      "original": {
        "owner": "nlewo",
        "repo": "comin",
        "type": "github"
      }
    },
    "deploy-rs": {
      "inputs": {
        "flake-compat": "flake-compat_2",
        "nixpkgs": [
          "nixpkgs"
        ],
        "utils": "utils"
      },
      "locked": {
        "lastModified": 1770019181,
        "narHash": "sha256-hwsYgDnby50JNVpTRYlF3UR/Rrpt01OrxVuryF40CFY=",
        "owner": "serokell",
        "repo": "deploy-rs",
        "rev": "77c906c0ba56aabdbc72041bf9111b565cdd6171",
        "type": "github"
      },
      "original": {
        "owner": "serokell",
        "repo": "deploy-rs",
        "type": "github"
      }
    },
    "devshell": {
      "inputs": {
        "nixpkgs": [
          "headplane",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1768818222,
        "narHash": "sha256-460jc0+CZfyaO8+w8JNtlClB2n4ui1RbHfPTLkpwhU8=",
        "owner": "numtide",
        "repo": "devshell",
        "rev": "255a2b1725a20d060f566e4755dbf571bbbb5f76",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "devshell",
        "type": "github"
      }
    },
    "flake-compat": {
      "flake": false,
      "locked": {
        "lastModified": 1765121682,
        "narHash": "sha256-4VBOP18BFeiPkyhy9o4ssBNQEvfvv1kXkasAYd0+rrA=",
        "owner": "NixOS",
        "repo": "flake-compat",
        "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "repo": "flake-compat",
        "type": "github"
      }
    },
    "flake-compat_2": {
      "flake": false,
      "locked": {
        "lastModified": 1733328505,
        "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
        "owner": "edolstra",
        "repo": "flake-compat",
        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
        "type": "github"
      },
      "original": {
        "owner": "edolstra",
        "repo": "flake-compat",
        "type": "github"
      }
    },
    "flake-compat_3": {
      "flake": false,
      "locked": {
        "lastModified": 1767039857,
        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
        "owner": "NixOS",
        "repo": "flake-compat",
        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "repo": "flake-compat",
        "type": "github"
      }
    },
    "flake-compat_4": {
      "flake": false,
      "locked": {
        "lastModified": 1767039857,
        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
        "owner": "NixOS",
        "repo": "flake-compat",
        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "repo": "flake-compat",
        "type": "github"
      }
    },
    "flake-parts": {
      "inputs": {
        "nixpkgs-lib": [
          "nixvim",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1768135262,
        "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=",
        "owner": "hercules-ci",
        "repo": "flake-parts",
        "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac",
        "type": "github"
      },
      "original": {
        "owner": "hercules-ci",
        "repo": "flake-parts",
        "type": "github"
      }
    },
    "flake-utils": {
      "inputs": {
        "systems": "systems_2"
      },
      "locked": {
        "lastModified": 1731533236,
        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    },
    "flake-utils_2": {
      "inputs": {
        "systems": "systems_3"
      },
      "locked": {
        "lastModified": 1731533236,
        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    },
    "git-hooks": {
      "inputs": {
        "flake-compat": "flake-compat_3",
        "gitignore": "gitignore",
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1772893680,
        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
        "owner": "cachix",
        "repo": "git-hooks.nix",
        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
        "type": "github"
      },
      "original": {
        "owner": "cachix",
        "repo": "git-hooks.nix",
        "type": "github"
      }
    },
    "git-hooks_2": {
      "inputs": {
        "flake-compat": [
          "nixos-mailserver",
          "flake-compat"
        ],
        "gitignore": "gitignore_2",
        "nixpkgs": [
          "nixos-mailserver",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1772893680,
        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
        "owner": "cachix",
        "repo": "git-hooks.nix",
        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
        "type": "github"
      },
      "original": {
        "owner": "cachix",
        "repo": "git-hooks.nix",
        "type": "github"
      }
    },
    "gitignore": {
      "inputs": {
        "nixpkgs": [
          "git-hooks",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1709087332,
        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
        "type": "github"
      },
      "original": {
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "type": "github"
      }
    },
    "gitignore_2": {
      "inputs": {
        "nixpkgs": [
          "nixos-mailserver",
          "git-hooks",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1709087332,
        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
        "type": "github"
      },
      "original": {
        "owner": "hercules-ci",
        "repo": "gitignore.nix",
        "type": "github"
      }
    },
    "headplane": {
      "inputs": {
        "devshell": "devshell",
        "flake-utils": "flake-utils",
        "nixpkgs": "nixpkgs_2"
      },
      "locked": {
        "lastModified": 1773108598,
        "narHash": "sha256-y80AABZv5n1vQua8mn1T79QB4pRnBTo+hPdmPa+J0yA=",
        "owner": "tale",
        "repo": "headplane",
        "rev": "6470f5a821e3ee5b4937a858bf13fb294bd38a7c",
        "type": "github"
      },
      "original": {
        "owner": "tale",
        "repo": "headplane",
        "type": "github"
      }
    },
    "ixx": {
      "inputs": {
        "flake-utils": [
          "nixvim",
          "nuschtosSearch",
          "flake-utils"
        ],
        "nixpkgs": [
          "nixvim",
          "nuschtosSearch",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1754860581,
        "narHash": "sha256-EM0IE63OHxXCOpDHXaTyHIOk2cNvMCGPqLt/IdtVxgk=",
        "owner": "NuschtOS",
        "repo": "ixx",
        "rev": "babfe85a876162c4acc9ab6fb4483df88fa1f281",
        "type": "github"
      },
      "original": {
        "owner": "NuschtOS",
        "ref": "v0.1.1",
        "repo": "ixx",
        "type": "github"
      }
    },
    "nixos-mailserver": {
      "inputs": {
        "blobs": "blobs",
        "flake-compat": "flake-compat_4",
        "git-hooks": "git-hooks_2",
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1773194666,
        "narHash": "sha256-YbsbqtTB3q0JjP7/G7GO58ea49cps1+8sb95/Bt7oVs=",
        "owner": "simple-nixos-mailserver",
        "repo": "nixos-mailserver",
        "rev": "489fbc4e0ef987cfdce700476abafe3269ebf3e5",
        "type": "gitlab"
      },
      "original": {
        "owner": "simple-nixos-mailserver",
        "repo": "nixos-mailserver",
        "type": "gitlab"
      }
    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1770107345,
        "narHash": "sha256-tbS0Ebx2PiA1FRW8mt8oejR0qMXmziJmPaU1d4kYY9g=",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "4533d9293756b63904b7238acb84ac8fe4c8c2c4",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "ref": "nixpkgs-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs-old-stable": {
      "locked": {
        "lastModified": 1767313136,
        "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "ref": "nixos-25.05",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs-unstable": {
      "locked": {
        "lastModified": 1772963539,
        "narHash": "sha256-9jVDGZnvCckTGdYT53d/EfznygLskyLQXYwJLKMPsZs=",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "9dcb002ca1690658be4a04645215baea8b95f31d",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "ref": "nixos-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_2": {
      "locked": {
        "lastModified": 1772736753,
        "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "917fec990948658ef1ccd07cef2a1ef060786846",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "ref": "nixpkgs-unstable",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixpkgs_3": {
      "locked": {
        "lastModified": 1773068389,
        "narHash": "sha256-vMrm7Pk2hjBRPnCSjhq1pH0bg350Z+pXhqZ9ICiqqCs=",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "44bae273f9f82d480273bab26f5c50de3724f52f",
        "type": "github"
      },
      "original": {
        "owner": "nixos",
        "ref": "nixos-25.11",
        "repo": "nixpkgs",
        "type": "github"
      }
    },
    "nixvim": {
      "inputs": {
        "flake-parts": "flake-parts",
        "nixpkgs": [
          "nixpkgs"
        ],
        "nuschtosSearch": "nuschtosSearch",
        "systems": "systems_4"
      },
      "locked": {
        "lastModified": 1769049374,
        "narHash": "sha256-h0Os2qqNyycDY1FyZgtbn28VF1ySP74/n0f+LDd8j+w=",
        "owner": "nix-community",
        "repo": "nixvim",
        "rev": "b8f76bf5751835647538ef8784e4e6ee8deb8f95",
        "type": "github"
      },
      "original": {
        "owner": "nix-community",
        "ref": "nixos-25.11",
        "repo": "nixvim",
        "type": "github"
      }
    },
    "nuschtosSearch": {
      "inputs": {
        "flake-utils": "flake-utils_2",
        "ixx": "ixx",
        "nixpkgs": [
          "nixvim",
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1768249818,
        "narHash": "sha256-ANfn5OqIxq3HONPIXZ6zuI5sLzX1sS+2qcf/Pa0kQEc=",
        "owner": "NuschtOS",
        "repo": "search",
        "rev": "b6f77b88e9009bfde28e2130e218e5123dc66796",
        "type": "github"
      },
      "original": {
        "owner": "NuschtOS",
        "repo": "search",
        "type": "github"
      }
    },
    "root": {
      "inputs": {
        "comin": "comin",
        "deploy-rs": "deploy-rs",
        "git-hooks": "git-hooks",
        "headplane": "headplane",
        "nixos-mailserver": "nixos-mailserver",
        "nixpkgs": "nixpkgs_3",
        "nixpkgs-old-stable": "nixpkgs-old-stable",
        "nixpkgs-unstable": "nixpkgs-unstable",
        "nixvim": "nixvim",
        "sops-nix": "sops-nix"
      }
    },
    "sops-nix": {
      "inputs": {
        "nixpkgs": [
          "nixpkgs"
        ]
      },
      "locked": {
        "lastModified": 1773096132,
        "narHash": "sha256-M3zEnq9OElB7zqc+mjgPlByPm1O5t2fbUrH3t/Hm5Ag=",
        "owner": "Mic92",
        "repo": "sops-nix",
        "rev": "d1ff3b1034d5bab5d7d8086a7803c5a5968cd784",
        "type": "github"
      },
      "original": {
        "owner": "Mic92",
        "repo": "sops-nix",
        "type": "github"
      }
    },
    "systems": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    },
    "systems_2": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    },
    "systems_3": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    },
    "systems_4": {
      "locked": {
        "lastModified": 1681028828,
        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
        "owner": "nix-systems",
        "repo": "default",
        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
        "type": "github"
      },
      "original": {
        "owner": "nix-systems",
        "repo": "default",
        "type": "github"
      }
    },
    "treefmt-nix": {
      "inputs": {
        "nixpkgs": "nixpkgs"
      },
      "locked": {
        "lastModified": 1770228511,
        "narHash": "sha256-wQ6NJSuFqAEmIg2VMnLdCnUc0b7vslUohqqGGD+Fyxk=",
        "owner": "numtide",
        "repo": "treefmt-nix",
        "rev": "337a4fe074be1042a35086f15481d763b8ddc0e7",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "treefmt-nix",
        "type": "github"
      }
    },
    "utils": {
      "inputs": {
        "systems": "systems"
      },
      "locked": {
        "lastModified": 1731533236,
        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
        "owner": "numtide",
        "repo": "flake-utils",
        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
        "type": "github"
      },
      "original": {
        "owner": "numtide",
        "repo": "flake-utils",
        "type": "github"
      }
    }
  },
  "root": "root",
  "version": 7
 }
--- a/flake.nix
+++ b/flake.nix
@ -10,7 +10,7 @@
    nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver";
    nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
-    headplane.url = "github:yrd/headplane-nix";
+    headplane.url = "github:tale/headplane";
    comin.url = "github:nlewo/comin";
    comin.inputs.nixpkgs.follows = "nixpkgs";
@ -41,7 +41,9 @@
      forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
-      lib = nixpkgs.lib;
+      # Extend nixpkgs.lib with our custom utils
      lib = nixpkgs.lib.extend (final: prev: self.lib or { });
      constants = import ./constants.nix;
      mkNixosConfiguration =
@ -59,6 +61,26 @@
        };
    in
    {
      # Custom library functions
      lib = {
        utils = import ./lib/utils.nix { lib = nixpkgs.lib; };
      };
      # Apps
      apps = forAllSystems (
        system:
        let
          pkgs = nixpkgs.legacyPackages.${system};
          mkApp = name: {
            type = "app";
            program = pkgs.lib.getExe (pkgs.callPackage ./apps/${name} { });
          };
        in
        {
          rebuild = mkApp "rebuild";
        }
      );
      packages = forAllSystems (system: import ./pkgs nixpkgs.legacyPackages.${system});
      overlays = import ./overlays { inherit inputs; };
--- a/hosts/cryodev-main/binfmt.nix
+++ b/hosts/cryodev-main/binfmt.nix
@ -0,0 +1,4 @@
 # Enable QEMU emulation for aarch64 to build Raspberry Pi images
 {
  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 }
--- a/hosts/cryodev-main/default.nix
+++ b/hosts/cryodev-main/default.nix
@ -1,11 +1,13 @@
 {
  inputs,
  lib,
  outputs,
  ...
 }:
 {
  imports = [
    ./binfmt.nix
    ./boot.nix
    ./hardware.nix
    ./networking.nix
@ -17,5 +19,12 @@
    outputs.nixosModules.nixvim
  ];
  # Allow unfree packages (netdata has changed to gpl3Plus ncul1 license)
  nixpkgs.config.allowUnfreePredicate =
    pkg:
    builtins.elem (lib.getName pkg) [
      "netdata"
    ];
  system.stateVersion = "25.11";
 }
--- a/hosts/cryodev-main/secrets.yaml
+++ b/hosts/cryodev-main/secrets.yaml
@ -0,0 +1,18 @@
 # SOPS encrypted secrets for cryodev-main
 # This file should be encrypted with sops before committing
 # See INSTRUCTIONS.md for setup instructions
 # Placeholder - replace with actual encrypted secrets
 forgejo-runner:
    token: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
 tailscale:
    auth-key: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
 headplane:
    cookie_secret: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
    agent_pre_authkey: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
 mailserver:
    accounts:
        forgejo: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
        admin: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
 forgejo:
    mail-pw: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
--- a/hosts/cryodev-main/services/forgejo.nix
+++ b/hosts/cryodev-main/services/forgejo.nix
@ -1,6 +1,5 @@
 {
  config,
  pkgs,
  outputs,
  constants,
  ...
@ -31,14 +30,17 @@
        USER = "forgejo@${constants.domain}";
      };
    };
    sops = true; # Enable sops integration for secrets
  };
  services.forgejo-runner = {
    enable = true;
    url = "https://${constants.services.forgejo.fqdn}";
-    # Token needs to be set up via sops/secrets
+    tokenFile = config.sops.secrets."forgejo-runner/token".path;
-    sops = true;
+  };
  sops.secrets."forgejo-runner/token" = {
    # gitea-runner user is created by gitea-actions-runner service
    mode = "0400";
  };
  services.nginx.virtualHosts."${constants.services.forgejo.fqdn}" = {
--- a/hosts/cryodev-main/services/headplane.nix
+++ b/hosts/cryodev-main/services/headplane.nix
@ -1,6 +1,4 @@
 {
  config,
  pkgs,
  outputs,
  constants,
  ...
@ -14,14 +12,11 @@
  services.headplane = {
    enable = true;
    port = constants.services.headplane.port;
-    headscale = {
+    settings = {
-      url = "http://127.0.0.1:${toString constants.services.headscale.port}";
+      headscale = {
-      public_url = "https://${constants.services.headscale.fqdn}";
+        url = "http://127.0.0.1:${toString constants.services.headscale.port}";
-    };
+        public_url = "https://${constants.services.headscale.fqdn}";
-    # Secrets for headplane need to be configured via sops
+      };
    sops.secrets = {
      "headplane/cookie_secret" = { };
      "headplane/agent_pre_authkey" = { };
    };
  };
--- a/hosts/cryodev-main/services/headscale.nix
+++ b/hosts/cryodev-main/services/headscale.nix
@ -1,6 +1,4 @@
 {
  config,
  pkgs,
  outputs,
  constants,
  ...
@ -17,7 +15,9 @@
    port = constants.services.headscale.port;
    settings = {
      server_url = "https://${constants.services.headscale.fqdn}";
-      dns_config.base_domain = constants.domain;
+      # dns.base_domain must be different from the server domain
      # Using "tail" for internal Tailscale DNS (e.g., host.tail)
      dns.base_domain = "tail";
    };
  };
--- a/hosts/cryodev-main/services/mailserver.nix
+++ b/hosts/cryodev-main/services/mailserver.nix
@ -1,6 +1,4 @@
 {
  config,
  pkgs,
  outputs,
  constants,
  ...
@ -21,7 +19,9 @@
        aliases = [ "postmaster" ];
      };
    };
-    certificateScheme = "acme-nginx";
+    x509.useACMEHost = constants.services.mail.fqdn;
    sops = true;
  };
  # ACME certificate for mail server
  security.acme.certs.${constants.services.mail.fqdn} = { };
 }
--- a/hosts/cryodev-pi/default.nix
+++ b/hosts/cryodev-pi/default.nix
@ -1,5 +1,6 @@
 {
  inputs,
  lib,
  outputs,
  ...
 }:
@ -10,12 +11,21 @@
    ./hardware.nix
    ./networking.nix
    ./packages.nix
    ./sd-image.nix
    ./services
    ./users.nix
    outputs.nixosModules.common
    outputs.nixosModules.nixvim
    outputs.nixosModules.sops
  ];
  # Allow unfree packages (netdata has changed to gpl3Plus ncul1 license)
  nixpkgs.config.allowUnfreePredicate =
    pkg:
    builtins.elem (lib.getName pkg) [
      "netdata"
    ];
  system.stateVersion = "25.11";
 }
--- a/hosts/cryodev-pi/sd-image.nix
+++ b/hosts/cryodev-pi/sd-image.nix
@ -0,0 +1,30 @@
 # SD Card image configuration for Raspberry Pi
 {
  config,
  modulesPath,
  lib,
  ...
 }:
 {
  imports = [
    (modulesPath + "/installer/sd-card/sd-image-aarch64.nix")
  ];
  sdImage = {
    # Compress with zstd for smaller download
    compressImage = true;
    # Auto-expand root partition on first boot
    expandOnBoot = true;
  };
  # Image filename based on hostname
  image.fileName = "${config.networking.hostName}-sd-image.img";
  # Disable ZFS to avoid build issues on SD image
  boot.supportedFilesystems = lib.mkForce [
    "vfat"
    "ext4"
  ];
 }
--- a/hosts/cryodev-pi/secrets.yaml
+++ b/hosts/cryodev-pi/secrets.yaml
@ -0,0 +1,11 @@
 # SOPS encrypted secrets for cryodev-pi
 # This file should be encrypted with sops before committing
 # See INSTRUCTIONS.md for setup instructions
 # Placeholder - replace with actual encrypted secrets
 # Generate UUID with: uuidgen
 netdata:
    stream:
        child-uuid: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
 tailscale:
    auth-key: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
--- a/hosts/cryodev-pi/services/netdata.nix
+++ b/hosts/cryodev-pi/services/netdata.nix
@ -1,7 +1,5 @@
 {
  config,
  pkgs,
  outputs,
  constants,
  ...
 }:
@ -9,23 +7,48 @@
 {
  services.netdata = {
    enable = true;
-    config = {
+    config.global = {
-      stream = {
+      "debug log" = "syslog";
-        enabled = "yes";
+      "access log" = "syslog";
-        destination = "${constants.hosts.cryodev-main.ip}:${toString constants.services.netdata.port}";
+      "error log" = "syslog";
-        "api key" = config.sops.placeholder."netdata/stream/child-uuid";
+    };
    configDir = {
      "stream.conf" = config.sops.templates."netdata/stream.conf".path;
    };
  };
  sops =
    let
      owner = config.services.netdata.user;
      group = config.services.netdata.group;
      mode = "0400";
      restartUnits = [ "netdata.service" ];
    in
    {
      # generate with `uuidgen`
      secrets."netdata/stream/child-uuid" = {
        inherit
          owner
          group
          mode
          restartUnits
          ;
      };
      templates."netdata/stream.conf" = {
        inherit
          owner
          group
          mode
          restartUnits
          ;
        # child node
        content = ''
          [stream]
          enabled = yes
          destination = ${constants.hosts.cryodev-main.ip}:${builtins.toString constants.services.netdata.port}
          api key = ${config.sops.placeholder."netdata/stream/child-uuid"}
        '';
      };
    };
  };
  # Make sure sops is enabled/imported for this host to handle the secret
  imports = [ outputs.nixosModules.sops ];
  sops = {
    defaultSopsFile = ../secrets.yaml;
    secrets."netdata/stream/child-uuid" = {
      owner = "netdata";
      group = "netdata";
    };
  };
 }
--- a/lib/utils.nix
+++ b/lib/utils.nix
@ -0,0 +1,85 @@
 { lib, ... }:
 let
  inherit (lib)
    mkDefault
    mkEnableOption
    mkIf
    mkOption
    types
    ;
 in
 {
  isNotEmptyStr = str: builtins.isString str && str != "";
  mkMailIntegrationOption = service: {
    enable = mkEnableOption "Mail integration for ${service}.";
    smtpHost = mkOption {
      type = types.str;
      default = "localhost";
      description = "SMTP host for sending emails.";
    };
  };
  mkReverseProxyOption = service: subdomain: {
    enable = mkEnableOption "Nginx reverse proxy for ${service}.";
    subdomain = mkOption {
      type = types.str;
      default = subdomain;
      description = "Subdomain for Nginx virtual host. Leave empty for root domain.";
    };
    forceSSL = mkOption {
      type = types.bool;
      default = true;
      description = "Force SSL for Nginx virtual host.";
    };
  };
  mkUrl =
    {
      fqdn,
      ssl ? false,
      port ? null,
      path ? "",
      ...
    }:
    let
      protocol = if ssl then "https" else "http";
      portPart = if port != null then ":${toString port}" else "";
      pathPart = if path != "" then "/${path}" else "";
    in
    "${protocol}://${fqdn}${portPart}${pathPart}";
  mkVirtualHost =
    {
      address ? "127.0.0.1",
      port ? null,
      socketPath ? null,
      location ? "/",
      ssl ? false,
      proxyWebsockets ? true,
      recommendedProxySettings ? true,
      extraConfig ? "",
      ...
    }:
    let
      target =
        if port != null then
          "http://${address}:${builtins.toString port}"
        else if socketPath != null then
          "http://unix:${socketPath}"
        else
          null;
    in
    {
      enableACME = ssl;
      forceSSL = ssl;
      locations = mkIf (target != null) {
        "${location}" = {
          proxyPass = mkDefault target;
          inherit proxyWebsockets recommendedProxySettings extraConfig;
        };
      };
    };
 }
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -3,6 +3,8 @@
  comin = import ./comin;
  forgejo = import ./forgejo;
  forgejo-runner = import ./forgejo-runner;
  headplane = import ./headplane;
  headscale = import ./headscale;
  mailserver = import ./mailserver;
  nixvim = import ./nixvim;
  normalUsers = import ./normalUsers;
--- a/modules/nixos/forgejo/default.nix
+++ b/modules/nixos/forgejo/default.nix
@ -18,31 +18,31 @@ in
 {
  config = mkIf cfg.enable {
    services.forgejo = {
-      database.type = "postgres";
+      database.type = mkDefault "postgres";
-      lfs.enable = true;
+      lfs.enable = mkDefault true;
      settings = {
        server = {
-          DOMAIN = "git.${config.networking.domain}";
+          DOMAIN = mkDefault "git.${config.networking.domain}";
-          PROTOCOL = "http";
+          PROTOCOL = mkDefault "http";
-          ROOT_URL = "https://${settings.server.DOMAIN}/";
+          ROOT_URL = mkDefault "https://${settings.server.DOMAIN}/";
-          HTTP_ADDR = "0.0.0.0";
+          HTTP_ADDR = mkDefault "0.0.0.0";
-          HTTP_PORT = 3456;
+          HTTP_PORT = mkDefault 3456;
-          SSH_PORT = head config.services.openssh.ports;
+          SSH_PORT = mkDefault (head config.services.openssh.ports);
        };
        service = {
-          DISABLE_REGISTRATION = true;
+          DISABLE_REGISTRATION = mkDefault true;
        };
        ui = {
-          DEFAULT_THEME = "forgejo-dark";
+          DEFAULT_THEME = mkDefault "forgejo-dark";
        };
        actions = {
-          ENABLED = true;
+          ENABLED = mkDefault true;
        };
        mailer = {
          ENABLED = mkDefault false;
-          SMTP_ADDR = "mail.${config.networking.domain}";
+          SMTP_ADDR = mkDefault "mail.${config.networking.domain}";
-          FROM = "git@${settings.server.DOMAIN}";
+          FROM = mkDefault "git@${settings.server.DOMAIN}";
-          USER = "git@${settings.server.DOMAIN}";
+          USER = mkDefault "git@${settings.server.DOMAIN}";
        };
      };
      secrets = {
--- a/modules/nixos/headplane/default.nix
+++ b/modules/nixos/headplane/default.nix
@ -2,31 +2,30 @@
  inputs,
  config,
  lib,
  pkgs,
  ...
 }:
 let
  cfg = config.services.headplane;
  domain = config.networking.domain;
  subdomain = cfg.reverseProxy.subdomain;
  fqdn = if (cfg.reverseProxy.enable && subdomain != "") then "${subdomain}.${domain}" else domain;
  headscale = config.services.headscale;
  inherit (lib)
    mkDefault
    mkIf
-    ;
+    mkOption
-
+    types
  inherit (lib.utils)
    mkReverseProxyOption
    mkVirtualHost
    ;
 in
 {
  imports = [ inputs.headplane.nixosModules.headplane ];
  options.services.headplane = {
-    reverseProxy = mkReverseProxyOption "Headplane" "hp";
+    port = mkOption {
      type = types.port;
      default = 3000;
      description = "Port for headplane to listen on";
    };
  };
  config = mkIf cfg.enable {
@ -37,14 +36,14 @@ in
    services.headplane = {
      settings = {
        server = {
-          host = mkDefault (if cfg.reverseProxy.enable then "127.0.0.1" else "0.0.0.0");
+          host = mkDefault "127.0.0.1";
-          port = mkDefault 3000;
+          port = mkDefault cfg.port;
          cookie_secret_path = config.sops.secrets."headplane/cookie_secret".path;
        };
        headscale = {
-          url = "http://127.0.0.1:${toString headscale.port}";
+          url = mkDefault "http://127.0.0.1:${toString headscale.port}";
-          public_url = headscale.settings.server_url;
+          public_url = mkDefault headscale.settings.server_url;
-          config_path = "/etc/headscale/config.yaml";
+          config_path = mkDefault "/etc/headscale/config.yaml";
        };
        integration.agent = {
          enabled = mkDefault true;
@ -53,13 +52,6 @@ in
      };
    };
    services.nginx.virtualHosts = mkIf cfg.reverseProxy.enable {
      "${fqdn}" = mkVirtualHost {
        port = cfg.settings.server.port;
        ssl = cfg.reverseProxy.forceSSL;
      };
    };
    sops.secrets =
      let
        owner = headscale.user;
--- a/modules/nixos/headscale/default.nix
+++ b/modules/nixos/headscale/default.nix
@ -65,24 +65,26 @@ in
      address = mkDefault (if cfg.reverseProxy.enable then "127.0.0.1" else "0.0.0.0");
      port = mkDefault 8077;
      settings = {
-        policy.path = "/etc/${acl}";
+        policy.path = mkDefault "/etc/${acl}";
-        database.type = "sqlite"; # postgres is highly discouraged as it is only supported for legacy reasons
+        database.type = mkDefault "sqlite"; # postgres is highly discouraged as it is only supported for legacy reasons
-        server_url = mkUrl {
+        server_url = mkDefault (mkUrl {
          inherit fqdn;
          ssl = with cfg.reverseProxy; enable && forceSSL;
-        };
+        });
-        derp.server.enable = cfg.reverseProxy.forceSSL;
+        derp.server.enable = mkDefault cfg.reverseProxy.forceSSL;
        dns = {
          magic_dns = mkDefault true;
          base_domain = mkDefault "tail";
-          search_domains = [ cfg.settings.dns.base_domain ];
+          search_domains = mkDefault [ cfg.settings.dns.base_domain ];
          override_local_dns = mkDefault true;
-          nameservers.global = optionals cfg.settings.dns.override_local_dns [
+          nameservers.global = mkDefault (
-            "1.1.1.1"
+            optionals cfg.settings.dns.override_local_dns [
-            "1.0.0.1"
+              "1.1.1.1"
-            "2606:4700:4700::1111"
+              "1.0.0.1"
-            "2606:4700:4700::1001"
+              "2606:4700:4700::1111"
-          ];
+              "2606:4700:4700::1001"
            ]
          );
        };
      };
    };
--- a/modules/nixos/mailserver/default.nix
+++ b/modules/nixos/mailserver/default.nix
@ -66,8 +66,10 @@ in
      fqdn = mkDefault fqdn;
      domains = mkDefault [ domain ];
-      certificateScheme = mkDefault "acme-nginx";
+      # stateVersion 3 requires the new mail directory structure
-      stateVersion = mkDefault 1;
+      # For new installations, this is the correct value
      # For existing installations, see: https://nixos-mailserver.readthedocs.io/en/latest/migrations.html
      stateVersion = mkDefault 3;
      loginAccounts = mapAttrs' (
        user: accConf:
@ -79,8 +81,14 @@ in
          hashedPasswordFile = config.sops.secrets."mailserver/accounts/${user}".path;
        }
      ) cfg.accounts;
      # Use ACME for certificate
      x509.useACMEHost = mkDefault fqdn;
    };
    # ACME certificate for mail server
    security.acme.certs.${fqdn} = { };
    security.acme = {
      acceptTerms = true;
      defaults.email = mkDefault "postmaster@cryodev.xyz";
--- a/modules/nixos/nixvim/default.nix
+++ b/modules/nixos/nixvim/default.nix
@ -15,7 +15,9 @@ in
    inputs.nixvim.nixosModules.nixvim
    ./plugins
-    ./spellfiles.nix
+    # TODO: spellfiles.nix uses home-manager options (home.file, xdg.dataHome)
    # which are not available in NixOS modules. Needs to be rewritten.
    # ./spellfiles.nix
  ];
  config = {
--- a/modules/nixos/nixvim/plugins/lsp.nix
+++ b/modules/nixos/nixvim/plugins/lsp.nix
@ -64,6 +64,6 @@ in
      };
    };
-    home.packages = optional (cfg.enable && plugin.servers.nixd.enable) pkgs.nixfmt;
+    environment.systemPackages = optional (cfg.enable && plugin.servers.nixd.enable) pkgs.nixfmt;
  };
 }
--- a/modules/nixos/nixvim/plugins/telescope.nix
+++ b/modules/nixos/nixvim/plugins/telescope.nix
@ -45,7 +45,7 @@ in
      ];
    };
-    home.packages = optionals plugin.enable [
+    environment.systemPackages = optionals plugin.enable [
      pkgs.ripgrep # for "live_grep"
    ];
  };
--- a/modules/nixos/nixvim/plugins/treesitter.nix
+++ b/modules/nixos/nixvim/plugins/treesitter.nix
@ -30,12 +30,12 @@ in
    };
    # Fix for: ERROR `cc` executable not found.
-    home.sessionVariables = mkIf plugin.enable {
+    environment.sessionVariables = mkIf plugin.enable {
      CC = mkDefault cc;
    };
    # Fix for: WARNING `tree-sitter` executable not found
-    home.packages = mkIf plugin.enable [
+    environment.systemPackages = mkIf plugin.enable [
      plugin.package
    ];
  };
--- a/modules/nixos/sops/default.nix
+++ b/modules/nixos/sops/default.nix
@ -7,7 +7,16 @@
 }:
 let
-  secrets = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets/secrets.yaml";
+  # Check both locations for secrets.yaml
  secretsInSubdir = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets/secrets.yaml";
  secretsInRoot = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets.yaml";
  secrets =
    if builtins.pathExists secretsInSubdir then
      secretsInSubdir
    else if builtins.pathExists secretsInRoot then
      secretsInRoot
    else
      null;
 in
 {
  imports = [ inputs.sops-nix.nixosModules.sops ];
@ -17,5 +26,5 @@ in
    sops
  ];
-  sops.defaultSopsFile = lib.mkIf (builtins.pathExists secrets) (lib.mkDefault secrets);
+  sops.defaultSopsFile = lib.mkIf (secrets != null) (lib.mkDefault secrets);
 }