Add SD image pipeline, documentation overhaul, and fix module issues

- Add automatic SD image builds for Raspberry Pi via Forgejo Actions - Enable binfmt emulation on cryodev-main for aarch64 cross-builds - Add sd-image.nix module to cryodev-pi configuration - Create comprehensive docs/ structure with installation guides - Split installation docs into: first-install (server), reinstall, new-client (Pi) - Add lib/utils.nix and apps/rebuild from synix - Fix headplane module for new upstream API (tale/headplane) - Fix various module issues (mailserver stateVersion, option conflicts) - Add placeholder secrets.yaml files for both hosts - Remove old INSTRUCTIONS.md (content moved to docs/)
2026-03-11 08:41:58 +01:00 · 2026-03-11 08:41:58 +01:00 · 5ba78886d2
commit 5ba78886d2
parent a5261d8ff0
44 changed files with 3570 additions and 609 deletions
--- a/.forgejo/workflows/build-pi-image.yml
+++ b/.forgejo/workflows/build-pi-image.yml
@ -0,0 +1,97 @@
+name: Build Raspberry Pi SD Images
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'hosts/**'
+      - 'modules/**'
+      - 'templates/**'
+      - 'flake.nix'
+      - 'flake.lock'
+      - 'constants.nix'
+  workflow_dispatch:
+
+jobs:
+  build-pi-images:
+    runs-on: host
+    strategy:
+      matrix:
+        # Add new Pi hosts to this list when created
+        host: [cryodev-pi]
+      fail-fast: false
+    
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Build SD image for ${{ matrix.host }}
+        run: |
+          echo "Building SD image for: ${{ matrix.host }}"
+          echo "This may take 30-60 minutes with emulation..."
+          
+          nix build .#nixosConfigurations.${{ matrix.host }}.config.system.build.sdImage \
+            --extra-platforms aarch64-linux \
+            --out-link result-${{ matrix.host }}
+          
+          IMAGE_PATH=$(find result-${{ matrix.host }} -name "*.img.zst" -type f | head -1)
+          if [ -z "$IMAGE_PATH" ]; then
+            echo "Error: No image found!"
+            exit 1
+          fi
+          
+          cp "$IMAGE_PATH" ./${{ matrix.host }}-sd-image.img.zst
+          sha256sum ${{ matrix.host }}-sd-image.img.zst > ${{ matrix.host }}-sd-image.img.zst.sha256
+          
+          echo "Image size:"
+          ls -lh ${{ matrix.host }}-sd-image.img.zst
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.host }}-sd-image
+          path: |
+            ${{ matrix.host }}-sd-image.img.zst
+            ${{ matrix.host }}-sd-image.img.zst.sha256
+
+  create-release:
+    needs: build-pi-images
+    runs-on: host
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts/
+
+      - name: Create Release and Upload
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          VERSION="v$(date +%Y-%m-%d)-$(git rev-parse --short HEAD)"
+          
+          # Create release via API
+          curl -s -X POST \
+            -H "Authorization: token ${GITHUB_TOKEN}" \
+            -H "Content-Type: application/json" \
+            -d "{\"tag_name\": \"${VERSION}\", \"name\": \"Pi Images ${VERSION}\", \"body\": \"Raspberry Pi SD card images. See docs for usage.\", \"draft\": false, \"prerelease\": false}" \
+            "https://git.cryodev.xyz/api/v1/repos/${GITHUB_REPOSITORY}/releases" \
+            -o release.json
+          
+          RELEASE_ID=$(jq -r '.id' release.json)
+          echo "Release ID: $RELEASE_ID"
+          
+          # Upload all files
+          for file in $(find artifacts -type f); do
+            echo "Uploading: $(basename $file)"
+            curl -s -X POST \
+              -H "Authorization: token ${GITHUB_TOKEN}" \
+              -H "Content-Type: application/octet-stream" \
+              --data-binary @"$file" \
+              "https://git.cryodev.xyz/api/v1/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets?name=$(basename $file)"
+          done
+          
+          echo "Done: https://git.cryodev.xyz/${GITHUB_REPOSITORY}/releases/tag/${VERSION}"
--- a/INSTRUCTIONS.md
+++ b/INSTRUCTIONS.md
@ -1,391 +0,0 @@
-# Server Setup Instructions / Server-Einrichtungsanleitung
-
---
-
-# 🇬🇧 English Instructions
-
-## 1. Prerequisites
-
-Ensure you have the following tools installed on your local machine:
-   `nix` (with flakes enabled)
-   `sops`
-   `age`
-   `ssh`
-
-## 2. DNS Configuration
-
-Configure the following DNS records for your domain `cryodev.xyz`:
-
-| Hostname | Type | Value | Purpose |
-|----------|------|-------|---------|
-| `@` | A | `<SERVER_IP>` | Main entry point |
-| `@` | AAAA | `<SERVER_IPV6>` | Main entry point (IPv6) |
-| `git` | CNAME | `@` | Forgejo |
-| `headscale` | CNAME | `@` | Headscale |
-| `headplane` | CNAME | `@` | Headplane |
-| `netdata` | CNAME | `@` | Netdata Monitoring |
-| `mail` | A | `<SERVER_IP>` | Mailserver |
-| `mail` | AAAA | `<SERVER_IPV6>` | Mailserver (IPv6) |
-| `@` | MX | `10 mail.cryodev.xyz.` | Mail delivery |
-| `@` | TXT | `"v=spf1 mx ~all"` | SPF Record |
-| `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC Record |
-
-## 3. Secret Management (SOPS)
-
-This repository uses `sops-nix` to manage secrets encrypted with `age`, utilizing the SSH host keys of the servers.
-
-### 3.1 Get Server Public Keys
-You need to convert the servers' SSH host public keys to age public keys.
-
-**For `cryodev-main`:**
-```bash
-nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <MAIN_IP> | ssh-to-age'
-```
-
-**For `cryodev-pi`:**
-```bash
-nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <PI_IP> | ssh-to-age'
-```
-
-### 3.2 Configure `.sops.yaml`
-Edit the `.sops.yaml` file in the root of this repository. Add the age public keys to the `keys` section and ensure creation rules exist for both hosts.
-
-```yaml
-keys:
-  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t # Admin Key (Steffen)
-  - &main_key age1... # cryodev-main Key
-  - &pi_key age1...   # cryodev-pi Key
-creation_rules:
-  - path_regex: hosts/cryodev-main/secrets.yaml$
-    key_groups:
-      - age:
-        - *admin_key
-        - *main_key
-  - path_regex: hosts/cryodev-pi/secrets.yaml$
-    key_groups:
-      - age:
-        - *admin_key
-        - *pi_key
-```
-
-### 3.3 Generating Secret Values
-
-**Mailserver Passwords (for `cryodev-main`):**
-```bash
-nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'
-```
-
-**Headplane Secrets (for `cryodev-main`):**
-```bash
-nix-shell -p openssl --run "openssl rand -hex 16"
-# Agent Pre-Authkey requires Headscale running:
-sudo headscale users create headplane-agent
-sudo headscale preauthkeys create --expiration 99y --reusable --user headplane-agent
-```
-
-**Tailscale Auth Keys (for both hosts):**
-*Requires Headscale running on `cryodev-main`.*
-```bash
-# For cryodev-main:
-sudo headscale preauthkeys create --expiration 99y --reusable --user default
-# For cryodev-pi:
-sudo headscale preauthkeys create --expiration 99y --reusable --user default
-```
-
-**Netdata Child UUID (for `cryodev-pi`):**
-```bash
-uuidgen
-```
-
-**Forgejo Runner Token:**
-Get from Forgejo Admin Panel.
-
-### 3.4 Creating Secrets Files
-
-**`hosts/cryodev-main/secrets.yaml`:**
-```bash
-sops hosts/cryodev-main/secrets.yaml
-```
-```yaml
-mailserver:
-    accounts:
-        forgejo: "$2y$05$..."
-        admin: "$2y$05$..."
-
-forgejo-runner:
-    token: "..."
-
-headplane:
-    cookie_secret: "..."
-    agent_pre_authkey: "..."
-
-tailscale:
-    auth-key: "..."
-```
-
-**`hosts/cryodev-pi/secrets.yaml`:**
-```bash
-sops hosts/cryodev-pi/secrets.yaml
-```
-```yaml
-tailscale:
-    auth-key: "..."
-
-netdata:
-    stream:
-        child-uuid: "..." # Output from uuidgen
-```
-
-## 4. Initial Deployment (Bootstrap)
-
-Before the continuous deployment can take over, you must perform an initial deployment manually using the provided install script.
-
-### 4.1 Prepare Target Machine
-1.  Boot into the NixOS Installation ISO.
-2.  Set a root password (for SSH): `passwd`.
-3.  Ensure internet connectivity.
-
-### 4.2 Run Install Script
-From your local machine (where this repo is), copy the script to the target or run it directly if you can fetch it.
-
-**Method A: Copy Script via SSH**
-```bash
-scp scripts/install.sh nixos@<TARGET_IP>:install.sh
-ssh nixos@<TARGET_IP>
-sudo -i
-chmod +x /home/nixos/install.sh
-./home/nixos/install.sh -r <GIT_REPO_URL> -n <HOSTNAME>
-```
-
-**Method B: Run on Target (if repo is public or reachable)**
-```bash
-# On the target machine (as root)
-nix-shell -p git
-git clone <GIT_REPO_URL> /tmp/nixos
-cd /tmp/nixos
-bash scripts/install.sh -n <HOSTNAME>
-```
-
-*Note: The script handles disk partitioning (via disko/script), hardware config generation, and installation.*
-
-## 5. Continuous Deployment (CD)
-
-### 5.1 cryodev-pi (Pull-based via Comin)
-The `cryodev-pi` host is configured to pull updates automatically via `comin`.
-
-1.  **Create Repository:** Create a new repository named `cryodev-server` on your Forgejo instance (`https://git.cryodev.xyz`).
-2.  **Push Configuration:** Push this entire NixOS configuration to the `main` branch of that repository.
-3.  **Comin URL:** The configuration expects the repository at: `https://git.cryodev.xyz/steffen/cryodev-server.git`.
-
-### 5.2 cryodev-main (Push-based via Forgejo Actions)
-The main server is deployed via a Forgejo Action.
-
-1.  **Generate SSH Key:**
-    ```bash
-    ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
-    ```
-2.  **Add Public Key:** Add the content of `deploy_key.pub` to `/root/.ssh/authorized_keys` on `cryodev-main`.
-3.  **Add Secret:** Add the content of `deploy_key` (private key) as a secret named `DEPLOY_SSH_KEY` in your Forgejo repository settings.
-
-## 6. Creating New Hosts (Templates)
-
-To quickly bootstrap a new host configuration, you can use the provided templates.
-
-1.  **Copy Template:**
-    ```bash
-    # For a Raspberry Pi:
-    cp -r templates/raspberry-pi hosts/new-pi-name
-
-    # For a generic x86 server:
-    cp -r templates/generic-server hosts/new-server-name
-    ```
-2.  **Adjust Configuration:**
-    *   **Hostname:** Edit `hosts/new-name/networking.nix`.
-    *   **Flake:** Register the new host in `flake.nix` under `nixosConfigurations`.
-    *   **Constants:** Add IP and ports to `constants.nix`.
-    *   **Secrets:** Add keys to `.sops.yaml` and create `hosts/new-name/secrets.yaml`.
-
---
-
-# 🇩🇪 Deutsche Anleitung
-
-## 1. Voraussetzungen
-
-Stellen Sie sicher, dass folgende Tools lokal installiert sind:
-   `nix` (mit Flakes)
-   `sops`
-   `age`
-   `ssh`
-   `ssh-to-age`
-   `uuidgen`
-
-## 2. DNS-Konfiguration
-
-Richten Sie folgende DNS-Einträge für `cryodev.xyz` ein:
-
-| Hostname | Typ | Wert | Zweck |
-|----------|-----|------|-------|
-| `@` | A | `<SERVER_IP>` | Hauptserver |
-| `@` | AAAA | `<SERVER_IPV6>` | Hauptserver (IPv6) |
-| `git` | CNAME | `@` | Forgejo |
-| `headscale` | CNAME | `@` | Headscale |
-| `headplane` | CNAME | `@` | Headplane |
-| `netdata` | CNAME | `@` | Netdata Monitoring |
-| `mail` | A | `<SERVER_IP>` | Mailserver |
-| `mail` | AAAA | `<SERVER_IPV6>` | Mailserver (IPv6) |
-| `@` | MX | `10 mail.cryodev.xyz.` | Mail-Empfang |
-| `@` | TXT | `"v=spf1 mx ~all"` | SPF-Record |
-| `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC-Record |
-
-## 3. Verwaltung von Geheimnissen (SOPS)
-
-Dieses Repository nutzt `sops-nix` mit den SSH-Host-Keys der Server.
-
-### 3.1 Public Keys abrufen
-
-**Für `cryodev-main`:**
-```bash
-nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <MAIN_IP> | ssh-to-age'
-```
-
-**Für `cryodev-pi`:**
-```bash
-nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <PI_IP> | ssh-to-age'
-```
-
-### 3.2 `.sops.yaml` konfigurieren
-Bearbeiten Sie `.sops.yaml` und fügen Sie die Keys sowie Regeln für beide Hosts hinzu:
-
-```yaml
-keys:
-  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t # Admin Key (Steffen)
-  - &main_key age1... # cryodev-main Key
-  - &pi_key age1...   # cryodev-pi Key
-creation_rules:
-  - path_regex: hosts/cryodev-main/secrets.yaml$
-    key_groups:
-      - age:
-        - *admin_key
-        - *main_key
-  - path_regex: hosts/cryodev-pi/secrets.yaml$
-    key_groups:
-      - age:
-        - *admin_key
-        - *pi_key
-```
-
-### 3.3 Werte generieren
-
-**Mailserver:** `mkpasswd -sm bcrypt`
-**Headplane:** `openssl rand -hex 16`
-**Netdata UUID:** `uuidgen`
-
-**Tailscale Auth Keys (auf `cryodev-main`):**
-```bash
-sudo headscale preauthkeys create --expiration 99y --reusable --user default
-```
-
-### 3.4 Secrets-Dateien erstellen
-
-**`hosts/cryodev-main/secrets.yaml`:**
-```bash
-sops hosts/cryodev-main/secrets.yaml
-```
-```yaml
-mailserver:
-    accounts:
-        forgejo: "$2y$05$..."
-        admin: "$2y$05$..."
-
-forgejo-runner:
-    token: "..."
-
-headplane:
-    cookie_secret: "..."
-    agent_pre_authkey: "..."
-
-tailscale:
-    auth-key: "..."
-```
-
-**`hosts/cryodev-pi/secrets.yaml`:**
-```bash
-sops hosts/cryodev-pi/secrets.yaml
-```
-```yaml
-tailscale:
-    auth-key: "..."
-
-netdata:
-    stream:
-        child-uuid: "..." # Output von uuidgen
-```
-
-## 4. Erstes Deployment (Bootstrap)
-
-Bevor das automatische Deployment funktionieren kann, müssen Sie das System einmal manuell mit dem Installationsskript installieren.
-
-### 4.1 Zielmaschine vorbereiten
-1.  Booten Sie das NixOS Installations-ISO.
-2.  Setzen Sie ein Root-Passwort: `passwd`.
-3.  Stellen Sie eine Internetverbindung her.
-
-### 4.2 Install-Script ausführen
-Kopieren Sie das Skript von Ihrem lokalen Rechner auf das Zielsystem.
-
-**Methode A: Per SCP**
-```bash
-scp scripts/install.sh nixos@<TARGET_IP>:install.sh
-ssh nixos@<TARGET_IP>
-sudo -i
-chmod +x /home/nixos/install.sh
-./home/nixos/install.sh -r <GIT_REPO_URL> -n <HOSTNAME>
-```
-
-**Methode B: Direkt auf dem Ziel (bei öffentlichem/erreichbarem Repo)**
-```bash
-# Auf der Zielmaschine (als root)
-nix-shell -p git
-git clone <GIT_REPO_URL> /tmp/nixos
-cd /tmp/nixos
-bash scripts/install.sh -n <HOSTNAME>
-```
-
-*Hinweis: Das Skript kümmert sich um Partitionierung, Hardware-Config und Installation.*
-
-## 5. Continuous Deployment (CD)
-
-### 5.1 cryodev-pi (Pull-basiert via Comin)
-Der Host `cryodev-pi` zieht Updates automatisch via `comin`.
-
-1.  **Repository erstellen:** Erstellen Sie ein Repository namens `cryodev-server` auf `https://git.cryodev.xyz`.
-2.  **Konfiguration pushen:** Pushen Sie diese Konfiguration in den `main`-Branch.
-3.  **Comin URL:** `https://git.cryodev.xyz/steffen/cryodev-server.git`.
-
-### 5.2 cryodev-main (Push-basiert via Forgejo Actions)
-Der Hauptserver wird über eine Forgejo Action deployt.
-
-1.  **SSH Key generieren:**
-    ```bash
-    ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
-    ```
-2.  **Public Key hinzufügen:** Inhalt von `deploy_key.pub` in `/root/.ssh/authorized_keys` auf `cryodev-main` eintragen.
-3.  **Secret hinzufügen:** Inhalt von `deploy_key` (Private Key) als Secret `DEPLOY_SSH_KEY` im Forgejo-Repository hinterlegen.
-
-## 6. Neue Hosts erstellen (Templates)
-
-Um schnell eine neue Host-Konfiguration zu erstellen, können Sie die bereitgestellten Templates nutzen.
-
-1.  **Template kopieren:**
-    ```bash
-    # Für einen Raspberry Pi:
-    cp -r templates/raspberry-pi hosts/neuer-pi-name
-
-    # Für einen generischen x86 Server:
-    cp -r templates/generic-server hosts/neuer-server-name
-    ```
-2.  **Konfiguration anpassen:**
-    *   **Hostname:** Bearbeiten Sie `hosts/neuer-name/networking.nix`.
-    *   **Flake:** Registrieren Sie den neuen Host in `flake.nix` unter `nixosConfigurations`.
-    *   **Constants:** Fügen Sie IP und Ports in `constants.nix` hinzu.
-    *   **Secrets:** Fügen Sie Keys zu `.sops.yaml` hinzu und erstellen Sie `hosts/neuer-name/secrets.yaml`.
--- a/README.md
+++ b/README.md
@ -1,147 +1,100 @@
-# cryodev-server NixOS Configuration
+# cryodev NixOS Configuration

-This repository contains the declarative NixOS configuration for the **cryodev** infrastructure, managed using **Nix Flakes**. It defines a robust, secure, and self-hosted environment spanning a main server and client devices.
+Declarative NixOS infrastructure for the **cryodev** environment, managed with Nix Flakes.

---
+## Quick Start

-# 🇬🇧 English Description
+```bash
+# Clone repository
+git clone https://git.cryodev.xyz/steffen/cryodev-server.git
+cd cryodev-server

-## Overview
+# Check configuration
+nix flake check

-The infrastructure is designed around a central server (`cryodev-main`) and satellite clients (e.g., `cryodev-pi`). It leverages modern DevOps practices including Infrastructure as Code (IaC), GitOps, and Mesh VPNs.
+# Build a host
+nix build .#nixosConfigurations.cryodev-main.config.system.build.toplevel
+```

-## Key Features & Architecture
+## Hosts

-### 🖥️ Hosts
-*   **`cryodev-main` (x86_64 Server)**: The core infrastructure hub.
-*   **`cryodev-pi` (Raspberry Pi 4)**: A remote client/worker node.
+| Host | Architecture | Deployment | Description |
+|------|--------------|------------|-------------|
+| `cryodev-main` | x86_64 | Push (deploy-rs) | Main server |
+| `cryodev-pi` | aarch64 | Pull (Comin) | Raspberry Pi client |

-### 🚀 Continuous Deployment (CD)
-We utilize different deployment strategies optimized for each host type:
-*   **Push-based (Server):** The main server is deployed via **Forgejo Actions** using **[deploy-rs](https://github.com/serokell/deploy-rs)**. This ensures immediate updates and robust rollbacks in case of failure.
-*   **Pull-based (Client):** The Raspberry Pi uses **[Comin](https://github.com/nlewo/comin)** to periodically poll the Git repository and apply updates automatically. This is ideal for devices behind NAT or with unstable connections.
+## Services

-### 🌐 Networking (Tailscale & Headscale)
-*   **Self-hosted VPN:** We run **Headscale**, an open-source implementation of the Tailscale control server.
-*   **Headplane:** A web frontend for managing Headscale users and routes.
-*   **Mesh Network:** All hosts are connected via a secure, private WireGuard mesh network.
-*   **MagicDNS:** Automatic DNS resolution for devices within the tailnet.
+| Service | Domain | Description |
+|---------|--------|-------------|
+| Headscale | `headscale.cryodev.xyz` | Self-hosted Tailscale server |
+| Headplane | `headplane.cryodev.xyz` | Headscale web UI |
+| Forgejo | `git.cryodev.xyz` | Git hosting with CI/CD |
+| Netdata | `netdata.cryodev.xyz` | Monitoring dashboard |
+| Mail | `mail.cryodev.xyz` | Email (Postfix/Dovecot) |

-### 📊 Monitoring (Netdata)
-*   **Parent/Child Streaming:** The main server acts as a centralized Netdata parent node.
-*   **Distributed Monitoring:** `cryodev-pi` (and other clients) stream their metrics securely over the Tailscale VPN to the parent node.
-*   **Alerting:** Integrated with the mailserver to send health alerts.
+## Raspberry Pi SD Images

-### 📧 Mail Services
-*   **NixOS Mailserver:** A fully functional mail stack (Postfix/Dovecot).
-*   **Integration:** Used by internal services (Forgejo, Netdata) to send notifications.
-*   **Security:** SPF, DKIM, and DMARC configured for `cryodev.xyz`.
+SD card images for Raspberry Pi clients are **built automatically** on every push to `main`.

-### 🛠️ Development & Productivity
-*   **Forgejo:** Self-hosted Git service (fork of Gitea) with built-in CI/CD Actions.
-*   **Forgejo Runner:** Self-hosted runners executing the CI/CD pipelines.
-*   **Neovim:** A fully pre-configured Neovim environment (aliased as `v`) available system-wide via a custom NixOS module.
-*   **Secret Management:** **[sops-nix](https://github.com/Mic92/sops-nix)** encrypts secrets using Age and SSH host keys, ensuring no sensitive data is committed in plain text.
-*   **Templates:** Ready-to-use NixOS configurations for quickly bootstrapping new clients.
-    *   `#raspberry-pi`: Template for Raspberry Pi 4 clients.
-    *   `#generic-server`: Template for generic x86_64 servers.
-*   **Bootstrap Script:** An `install.sh` script automates disk partitioning (via disko) and system installation for new hosts.
+Download from: [Releases](https://git.cryodev.xyz/steffen/cryodev-server/releases)

-## 🚧 Roadmap & Missing Features
+```bash
+# Flash to SD card
+zstd -d cryodev-pi-sd-image.img.zst
+sudo dd if=cryodev-pi-sd-image.img of=/dev/sdX bs=4M status=progress
+```

-### BioSafe Gateway (Dual Ethernet)
-The Raspberry Pi hosts utilize a custom board with two Ethernet ports:
-*   **WAN:** Standard Internet connection.
-*   **LAN (`eth1`):** A dedicated local connection managed specifically by the **BioSafe Gateway App**.
-*   *Status:* The network configuration logic and the integration of the controlling app are currently missing.
+See [Adding a new Raspberry Pi](docs/getting-started/new-client.md) for the full workflow.

-### Closed Source Integration
-The **BioSafe Gateway App** is closed source.
-*   It needs to be added as a **private Flake input**.
-*   Authentication mechanism (e.g., access tokens via Secrets) to fetch this private input during the build process is not yet implemented.
+## Documentation

-### SD Card Image Pipeline
-Currently, the Pi requires manual setup.
-*   *Goal:* A CI/CD pipeline that builds a fully configured, flashable SD card image.
-*   *Usage:* Download image -> Flash to SD -> Insert in Pi -> Boot & Auto-connect.
+Full documentation is available in the [`docs/`](docs/index.md) directory:
+
+- [Prerequisites](docs/getting-started/prerequisites.md)
+- [New Raspberry Pi Client](docs/getting-started/new-client.md)
+- [SD Image Reference](docs/getting-started/sd-image.md)
+- [Server Installation](docs/getting-started/first-install.md)
+- [Reinstallation](docs/getting-started/reinstall.md)
+- [Services](docs/services/)
+- [Deployment](docs/deployment/cd.md)

 ## Directory Structure
-*   `flake.nix`: The entry point defining inputs (dependencies) and outputs (system configs).
-*   `hosts/`: Configuration specific to each machine.
-*   `modules/`: Reusable NixOS modules (custom or imported).
-*   `pkgs/`: Custom packages.
-*   `constants.nix`: Central source of truth for IPs, ports, and domains.
-*   `INSTRUCTIONS.md`: Detailed setup guide (DNS, SOPS, Initial Deployment).
-*   `AGENTS.md`: Guidelines for AI agents working on this repository.

---
+```
+.
+├── flake.nix           # Flake entry point
+├── constants.nix       # Central configuration
+├── hosts/              # Host configurations
+├── modules/            # Reusable NixOS modules
+├── pkgs/               # Custom packages
+├── overlays/           # Nixpkgs overlays
+├── templates/          # Host templates
+├── scripts/            # Helper scripts
+├── apps/               # Nix apps (rebuild)
+├── lib/                # Helper functions
+└── docs/               # Documentation
+```

-# 🇩🇪 Deutsche Beschreibung
+## Commands

-## Überblick
+```bash
+# Format code
+nix fmt

-Die Infrastruktur ist um einen zentralen Server (`cryodev-main`) und Satelliten-Clients (z.B. `cryodev-pi`) herum aufgebaut. Sie nutzt moderne DevOps-Praktiken wie Infrastructure as Code (IaC), GitOps und Mesh-VPNs.
+# Run checks
+nix flake check

-## Hauptfunktionen & Architektur
+# Update dependencies
+nix flake update

-### 🖥️ Hosts
-*   **`cryodev-main` (x86_64 Server)**: Der zentrale Infrastruktur-Knoten.
-*   **`cryodev-pi` (Raspberry Pi 4)**: Ein entfernter Client/Worker-Node.
+# Enter dev shell
+nix develop

-### 🚀 Continuous Deployment (CD)
-Wir verwenden unterschiedliche Deployment-Strategien, optimiert für den jeweiligen Host-Typ:
-*   **Push-basiert (Server):** Der Hauptserver wird über **Forgejo Actions** mittels **[deploy-rs](https://github.com/serokell/deploy-rs)** aktualisiert. Dies garantiert sofortige Updates und robuste Rollbacks bei Fehlern.
-*   **Pull-basiert (Client):** Der Raspberry Pi nutzt **[Comin](https://github.com/nlewo/comin)**, um das Git-Repository periodisch abzufragen und Updates automatisch anzuwenden. Ideal für Geräte hinter NAT oder mit instabilen Verbindungen.
+# Build Pi SD image locally
+nix build .#nixosConfigurations.cryodev-pi.config.system.build.sdImage
+```

-### 🌐 Netzwerk (Tailscale & Headscale)
-*   **Self-hosted VPN:** Wir betreiben **Headscale**, eine Open-Source-Implementierung des Tailscale-Kontrollservers.
-*   **Headplane:** Ein Web-Frontend zur Verwaltung von Headscale-Benutzern und Routen.
-*   **Mesh-Netzwerk:** Alle Hosts sind über ein sicheres, privates WireGuard-Mesh-Netzwerk verbunden.
-*   **MagicDNS:** Automatische Namensauflösung für Geräte innerhalb des Tailnets.
+## License

-### 📊 Monitoring (Netdata)
-*   **Parent/Child Streaming:** Der Hauptserver agiert als zentraler Netdata Parent-Node.
-*   **Verteiltes Monitoring:** `cryodev-pi` (und andere Clients) streamen ihre Metriken sicher über das Tailscale-VPN an den Parent-Node.
-*   **Alarmierung:** Integriert mit dem Mailserver zum Versenden von Gesundheitswarnungen.
-
-### 📧 Mail-Dienste
-*   **NixOS Mailserver:** Ein voll funktionsfähiger Mail-Stack (Postfix/Dovecot).
-*   **Integration:** Wird von internen Diensten (Forgejo, Netdata) für Benachrichtigungen genutzt.
-*   **Sicherheit:** SPF, DKIM und DMARC sind für `cryodev.xyz` konfiguriert.
-
-### 🛠️ Entwicklung & Produktivität
-*   **Forgejo:** Self-hosted Git-Dienst (Gitea-Fork) mit integrierten CI/CD Actions.
-*   **Forgejo Runner:** Eigene Runner, die die CI/CD-Pipelines ausführen.
-*   **Neovim:** Eine vollständig vorkonfigurierte Neovim-Umgebung (Alias `v`), die systemweit über ein eigenes NixOS-Modul bereitgestellt wird.
-*   **Secret Management:** **[sops-nix](https://github.com/Mic92/sops-nix)** verschlüsselt Geheimnisse mittels Age und SSH-Host-Keys, sodass keine sensiblen Daten im Klartext gespeichert werden.
-*   **Templates:** Vorgefertigte NixOS-Konfigurationen zum schnellen Aufsetzen neuer Clients.
-    *   `#raspberry-pi`: Template für Raspberry Pi 4 Clients.
-    *   `#generic-server`: Template für generische x86_64 Server.
-*   **Bootstrap Skript:** Ein `install.sh` Skript automatisiert die Disk-Partitionierung (via disko) und Systeminstallation für neue Hosts.
-
-## 🚧 Roadmap & Fehlende Funktionen
-
-### BioSafe Gateway (Dual-Ethernet)
-Die Raspberry Pi Hosts nutzen ein spezielles Board mit zwei Ethernet-Anschlüssen:
-*   **WAN:** Normale Internetverbindung.
-*   **LAN (`eth1`):** Eine dedizierte lokale Verbindung, die spezifisch durch die **BioSafe Gateway App** verwaltet wird.
-*   *Status:* Die Logik für die Netzwerkkonfiguration und die Integration der steuernden App fehlen aktuell.
-
-### Integration von Closed Source
-Die **BioSafe Gateway App** ist Closed Source.
-*   Sie muss als **privater Flake Input** hinzugefügt werden.
-*   Mechanismen zur Authentifizierung (z.B. Access Tokens via Secrets) für das Abrufen dieses privaten Inputs während des Build-Prozesses sind noch nicht implementiert.
-
-### Pipeline für SD-Karten-Images
-Momentan erfordert der Pi eine manuelle Einrichtung.
-*   *Ziel:* Eine CI/CD-Pipeline, die ein fertig konfiguriertes, flashbares SD-Karten-Image baut.
-*   *Nutzung:* Image herunterladen -> auf SD flashen -> in Pi stecken -> booten & automatisch verbinden.
-
-## Verzeichnisstruktur
-*   `flake.nix`: Der Einstiegspunkt, der Inputs (Abhängigkeiten) und Outputs (Systemkonfigurationen) definiert.
-*   `hosts/`: Maschinenspezifische Konfigurationen.
-*   `modules/`: Wiederverwendbare NixOS-Module (eigene oder importierte).
-*   `pkgs/`: Benutzerdefinierte Pakete.
-*   `constants.nix`: Zentrale Quelle für IPs, Ports und Domains.
-*   `INSTRUCTIONS.md`: Detaillierte Einrichtungsanleitung (DNS, SOPS, Initial Deployment).
-*   `AGENTS.md`: Richtlinien für KI-Agenten, die an diesem Repository arbeiten.
+Private repository.
--- a/apps/rebuild/default.nix
+++ b/apps/rebuild/default.nix
@ -0,0 +1,30 @@
+{
+  writeShellApplication,
+  coreutils,
+  gnugrep,
+  gnused,
+  home-manager,
+  hostname,
+  nix,
+  nixos-rebuild,
+  ...
+}:
+
+let
+  name = "rebuild";
+  text = builtins.readFile ./${name}.sh;
+in
+writeShellApplication {
+  inherit name text;
+  meta.mainProgram = name;
+
+  runtimeInputs = [
+    coreutils
+    gnugrep
+    gnused
+    home-manager
+    hostname
+    nix
+    nixos-rebuild
+  ];
+}
--- a/apps/rebuild/rebuild.sh
+++ b/apps/rebuild/rebuild.sh
@ -0,0 +1,246 @@
+# NixOS and standalone Home Manager rebuild script
+
+# Defaults
+FLAKE_PATH="$HOME/.config/nixos" # Default flake path
+HOME_USER="$(whoami)"            # Default username. Used to identify the Home Manager configuration
+NIXOS_HOST="$(hostname)"         # Default hostname. Used to identify the NixOS and Home Manager configuration
+BUILD_HOST=""                    # Default build host. Empty means localhost
+TARGET_HOST=""                   # Default target host. Empty means localhost
+UPDATE=0                         # Default to not update flake repositories
+UPDATE_INPUTS=""                 # Default list of inputs to update. Empty means all
+ROLLBACK=0                       # Default to not rollback
+SHOW_TRACE=0                     # Default to not show detailed error messages
+
+# Function to display the help message
+Help() {
+  echo "Wrapper script for 'nixos-rebuild switch' and 'home-manager switch' commands."
+  echo "Usage: rebuild <command> [OPTIONS]"
+  echo
+  echo "Commands:"
+  echo "  nixos                  Rebuild NixOS configuration"
+  echo "  home                   Rebuild Home Manager configuration"
+  echo "  all                    Rebuild both NixOS and Home Manager configurations"
+  echo "  help                   Show this help message"
+  echo
+  echo "Options (for NixOS and Home Manager):"
+  echo "  -H, --host <host>      Specify the hostname (as in 'nixosConfiguraions.<host>'). Default: $NIXOS_HOST"
+  echo "  -p, --path <path>      Set the path to the flake directory. Default: $FLAKE_PATH"
+  echo "  -U, --update [inputs]  Update all flake inputs. Optionally provide comma-separated list of inputs to update instead."
+  echo "  -r, --rollback         Don't build the new configuration, but use the previous generation instead"
+  echo "  -t, --show-trace       Show detailed error messages"
+  echo
+  echo "NixOS only options:"
+  echo "  -B, --build-host <user@example.com>   Use a remote host for building the configuration via SSH"
+  echo "  -T, --target-host <user@example.com>  Deploy the configuration to a remote host via SSH. If '--host' is specified, it will be used as the target host."
+  echo
+  echo "Home Manager only options:"
+  echo "  -u, --user <user>      Specify the username (as in 'homeConfigurations.<user>@<host>'). Default: $HOME_USER"
+}
+
+# Function to handle errors
+error() {
+  echo "Error: $1"
+  exit 1
+}
+
+# Function to rebuild NixOS configuration
+Rebuild_nixos() {
+  local FLAKE="$FLAKE_PATH#$NIXOS_HOST"
+
+  # Construct rebuild command
+  local CMD=("nixos-rebuild" "switch" "--sudo")
+  [[ -n "$TARGET_HOST" || -n "$BUILD_HOST" ]] && CMD+=("--ask-sudo-password")
+  CMD+=("--flake" "$FLAKE")
+  [ "$ROLLBACK" = 1 ] && CMD+=("--rollback")
+  [ "$SHOW_TRACE" = 1 ] && CMD+=("--show-trace")
+  [ -n "$BUILD_HOST" ] && CMD+=("--build-host" "$BUILD_HOST")
+  if [ "$NIXOS_HOST" != "$(hostname)" ] && [ -z "$TARGET_HOST" ]; then
+    TARGET_HOST="$NIXOS_HOST"
+    echo "Using '$TARGET_HOST' as target host."
+  fi
+  [ -n "$TARGET_HOST" ] && CMD+=("--target-host" "$TARGET_HOST")
+
+  # Rebuild NixOS configuration
+  if [ "$ROLLBACK" = 0 ]; then 
+    echo "Rebuilding NixOS configuration '$FLAKE'..." 
+  else
+    echo "Rolling back to last NixOS generation..."
+  fi
+
+  echo "Executing command: ${CMD[*]}"
+  "${CMD[@]}" || error "NixOS rebuild failed"
+  echo "NixOS rebuild completed successfully."
+}
+
+# Function to rebuild Home Manager configuration
+Rebuild_home() {
+  local FLAKE="$FLAKE_PATH#$HOME_USER@$NIXOS_HOST"
+
+  if [ -n "$BUILD_HOST" ] || [ -n "$TARGET_HOST" ]; then
+    error "Remote building is not supported for Home Manager."
+  fi
+
+  # Construct rebuild command
+  local CMD=()
+  if [ "$ROLLBACK" = 1 ]; then
+    local rollback_path
+    rollback_path=$(home-manager generations | sed -n '2p' | grep -o '/nix/store[^ ]*')
+    CMD+=("$rollback_path/activate")
+  else
+    CMD=("home-manager" "switch" "--flake" "$FLAKE")
+    [ "$SHOW_TRACE" = 1 ] && CMD+=("--show-trace")
+  fi
+
+  # Rebuild Home Manager configuration
+  if [ "$ROLLBACK" = 0 ]; then 
+    echo "Rebuilding Home Manager configuration '$FLAKE'..." 
+  else
+    echo "Rolling back to last Home Manager generation..."
+  fi
+
+  echo "Executing command: ${CMD[*]}"
+  "${CMD[@]}" || error "Home Manager rebuild failed"
+  echo "Home Manager rebuild completed successfully."
+}
+
+# Function to Update flake repositories
+Update() {
+  echo "Updating flake inputs..."
+
+  # Construct update command as an array
+  local CMD=("nix" "flake" "update" "--flake" "$FLAKE_PATH")
+  if [ -n "$UPDATE_INPUTS" ]; then
+    # Split comma-separated inputs and pass them to nix flake update
+    IFS=',' read -ra INPUTS <<< "$UPDATE_INPUTS"
+    for input in "${INPUTS[@]}"; do
+      CMD+=("$input")
+    done
+  fi
+
+  echo "Executing command: ${CMD[*]}"
+  "${CMD[@]}" || error "Failed to update flake repositories"
+  echo "Flake repositories updated successfully."
+}
+
+# Parse command-line options
+if [[ -z "${1:-}" ]]; then
+  echo "Error: No command specified. Printing help page."
+  Help
+  exit 1
+fi
+COMMAND=$1
+shift
+
+# Handle help command early
+if [ "$COMMAND" = "help" ] || [ "$COMMAND" = "--help" ] || [ "$COMMAND" = "-h" ]; then
+  Help
+  exit 0
+fi
+
+while [ $# -gt 0 ]; do
+  case "${1:-}" in
+    -H|--host)
+      if [ -n "${2:-}" ]; then
+        NIXOS_HOST="$2"
+        shift 2
+      else
+        error "-H|--host option requires an argument"
+      fi
+      ;;
+    -u|--user)
+      if [ -n "${2:-}" ]; then
+        HOME_USER="$2"
+        shift 2
+      else
+        error "-u|--user option requires an argument"
+      fi
+      ;;
+    -p|--path)
+      if [ -n "${2:-}" ]; then
+        FLAKE_PATH="$2"
+        shift 2
+      else
+        error "-p|--path option requires an argument"
+      fi
+      ;;
+    -U|--update)
+      UPDATE=1
+      # Check if next argument is a non-option
+      if [ $# -gt 1 ] && [ "${2#-}" = "${2:-}" ]; then
+        UPDATE_INPUTS="$2"
+        shift 2
+      else
+        shift
+      fi
+      ;;
+    -r|--rollback)
+      ROLLBACK=1
+      shift
+      ;;
+    -t|--show-trace)
+      SHOW_TRACE=1
+      shift
+      ;;
+    -B|--build-host)
+      if [ -n "${2:-}" ]; then
+        BUILD_HOST="$2"
+        shift 2
+      else
+        error "-B|--build-host option requires an argument"
+      fi
+      ;;
+    -T|--target-host)
+      if [ -n "${2:-}" ]; then
+        TARGET_HOST="$2"
+        shift 2
+      else
+        error "-T|--target-host option requires an argument"
+      fi
+      ;;
+    *)
+      echo "Error: Unknown option '$1'"
+      Help
+      exit 1
+      ;;
+  esac
+done
+
+# Check if script is run with sudo
+if [ "$EUID" -eq 0 ]; then
+  error "Do not run this script with sudo."
+fi
+
+# Check if flake path exists
+if [ ! -d "$FLAKE_PATH" ]; then
+  error "Flake path '$FLAKE_PATH' does not exist"
+fi
+
+# Ignore trailing slash in flake path
+FLAKE_PATH="${FLAKE_PATH%/}"
+
+# Check if flake.nix exists
+if [ ! -f "$FLAKE_PATH/flake.nix" ]; then
+  error "flake.nix does not exist in '$FLAKE_PATH'"
+fi
+
+# Execute updates and rebuilds based on the command
+[ "$UPDATE" = 1 ] && Update
+
+case "$COMMAND" in
+  nixos)
+    Rebuild_nixos
+    ;;
+  home)
+    Rebuild_home
+    ;;
+  all)
+    Rebuild_nixos
+    Rebuild_home
+    ;;
+  *)
+    echo "Error: Unknown command '$COMMAND'"
+    echo "Printing help page:"
+    Help
+    exit 1
+    ;;
+esac
--- a/docs/deployment/cd.md
+++ b/docs/deployment/cd.md
@ -0,0 +1,174 @@
+# Continuous Deployment
+
+The cryodev infrastructure uses two deployment strategies optimized for different host types.
+
+## Overview
+
+| Host | Strategy | Tool | Trigger |
+|------|----------|------|---------|
+| `cryodev-main` | Push-based | deploy-rs | Git push via Forgejo Actions |
+| `cryodev-pi` | Pull-based | Comin | Periodic polling |
+
+## Push-based Deployment (cryodev-main)
+
+### How It Works
+
+1. Developer pushes to `main` branch
+2. Forgejo Actions workflow triggers
+3. `deploy-rs` connects via SSH and deploys
+
+### Setup
+
+#### 1. Generate Deploy Key
+
+```bash
+ssh-keygen -t ed25519 -f deploy_key -C "forgejo-actions"
+```
+
+#### 2. Add Public Key to Server
+
+On `cryodev-main`:
+
+```bash
+echo "PUBLIC_KEY_CONTENT" >> /root/.ssh/authorized_keys
+```
+
+#### 3. Add Private Key to Forgejo
+
+1. Go to Repository Settings > Secrets
+2. Add secret named `DEPLOY_SSH_KEY`
+3. Paste the private key content
+
+#### 4. Workflow Configuration
+
+`.forgejo/workflows/deploy.yaml`:
+
+```yaml
+name: Deploy
+on:
+  push:
+    branches: [main]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cachix/install-nix-action@v24
+      - run: nix flake check
+
+  deploy:
+    needs: check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cachix/install-nix-action@v24
+      
+      - name: Setup SSH
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan cryodev-main >> ~/.ssh/known_hosts
+          
+      - name: Deploy
+        run: nix run github:serokell/deploy-rs -- .#cryodev-main
+```
+
+### Rollback
+
+deploy-rs automatically rolls back if the new configuration fails health checks.
+
+Manual rollback:
+
+```bash
+# List generations
+sudo nix-env -p /nix/var/nix/profiles/system --list-generations
+
+# Rollback to previous
+sudo nixos-rebuild switch --rollback
+```
+
+## Pull-based Deployment (cryodev-pi)
+
+### How It Works
+
+1. Comin periodically polls the Git repository
+2. On changes, it builds and activates the new configuration
+3. Works through NAT without incoming connections
+
+### Configuration
+
+```nix
+# hosts/cryodev-pi/services/comin.nix
+{
+  services.comin = {
+    enable = true;
+    remotes = [{
+      name = "origin";
+      url = "https://git.cryodev.xyz/steffen/cryodev-server.git";
+      branches.main.name = "main";
+    }];
+  };
+}
+```
+
+### Monitoring
+
+Check Comin status:
+
+```bash
+sudo systemctl status comin
+sudo journalctl -u comin -f
+```
+
+Force immediate update:
+
+```bash
+sudo systemctl restart comin
+```
+
+### Troubleshooting
+
+If Comin fails to build:
+
+```bash
+# Check logs
+sudo journalctl -u comin --since "1 hour ago"
+
+# Manual build test
+cd /var/lib/comin/repo
+nix build .#nixosConfigurations.cryodev-pi.config.system.build.toplevel
+```
+
+## Manual Deployment
+
+For hosts not using automated deployment:
+
+```bash
+# Build locally
+nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel
+
+# Deploy with nixos-rebuild
+nixos-rebuild switch --flake .#<hostname> --target-host root@<hostname>
+
+# Or using deploy-rs
+nix run github:serokell/deploy-rs -- .#<hostname>
+```
+
+## Testing Changes
+
+Before pushing, always verify:
+
+```bash
+# Check flake validity
+nix flake check
+
+# Build configuration (dry-run)
+nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel --dry-run
+
+# Full build
+nix build .#nixosConfigurations.<hostname>.config.system.build.toplevel
+```
--- a/docs/deployment/dns.md
+++ b/docs/deployment/dns.md
@ -0,0 +1,93 @@
+# DNS Configuration
+
+Required DNS records for the cryodev infrastructure.
+
+## Primary Domain (cryodev.xyz)
+
+### A/AAAA Records
+
+| Hostname | Type | Value | Purpose |
+|----------|------|-------|---------|
+| `@` | A | `<SERVER_IP>` | Main server |
+| `@` | AAAA | `<SERVER_IPV6>` | Main server (IPv6) |
+| `mail` | A | `<SERVER_IP>` | Mail server |
+| `mail` | AAAA | `<SERVER_IPV6>` | Mail server (IPv6) |
+
+### CNAME Records
+
+| Hostname | Type | Value | Purpose |
+|----------|------|-------|---------|
+| `git` | CNAME | `@` | Forgejo |
+| `headscale` | CNAME | `@` | Headscale |
+| `headplane` | CNAME | `@` | Headplane |
+| `netdata` | CNAME | `@` | Netdata Monitoring |
+
+### Mail Records
+
+| Hostname | Type | Value | Purpose |
+|----------|------|-------|---------|
+| `@` | MX | `10 mail.cryodev.xyz.` | Mail delivery |
+| `@` | TXT | `"v=spf1 mx ~all"` | SPF |
+| `_dmarc` | TXT | `"v=DMARC1; p=none"` | DMARC |
+| `mail._domainkey` | TXT | `"v=DKIM1; k=rsa; p=..."` | DKIM |
+
+## Getting the DKIM Key
+
+After deploying the mailserver, retrieve the DKIM public key:
+
+```bash
+sudo cat /var/dkim/cryodev.xyz.mail.txt
+```
+
+Add this as a TXT record for `mail._domainkey.cryodev.xyz`.
+
+## Verification
+
+### Check DNS Propagation
+
+```bash
+# A record
+dig A cryodev.xyz
+
+# MX record
+dig MX cryodev.xyz
+
+# SPF
+dig TXT cryodev.xyz
+
+# DKIM
+dig TXT mail._domainkey.cryodev.xyz
+
+# DMARC
+dig TXT _dmarc.cryodev.xyz
+```
+
+### Online Tools
+
+- [MXToolbox](https://mxtoolbox.com/) - Comprehensive DNS/mail testing
+- [Mail-tester](https://www.mail-tester.com/) - Email deliverability testing
+- [DMARC Analyzer](https://dmarcanalyzer.com/) - DMARC record validation
+
+## TTL Recommendations
+
+For initial setup, use low TTLs (300 seconds) to allow quick changes.
+
+After verification, increase to:
+- A/AAAA records: 3600 (1 hour)
+- CNAME records: 3600 (1 hour)  
+- MX records: 3600 (1 hour)
+- TXT records: 3600 (1 hour)
+
+## Firewall Requirements
+
+Ensure these ports are open on `cryodev-main`:
+
+| Port | Protocol | Service |
+|------|----------|---------|
+| 22 | TCP | SSH |
+| 80 | TCP | HTTP (ACME/redirect) |
+| 443 | TCP | HTTPS |
+| 25 | TCP | SMTP |
+| 465 | TCP | SMTPS |
+| 587 | TCP | SMTP Submission |
+| 993 | TCP | IMAPS |
--- a/docs/getting-started/first-install.md
+++ b/docs/getting-started/first-install.md
@ -0,0 +1,179 @@
+# Erstinstallation (x86_64 Server)
+
+Diese Anleitung beschreibt die **manuelle Installation** eines neuen x86_64 Servers (z.B. cryodev-main).
+
+> **Für Raspberry Pi:** Siehe [Neuen Raspberry Pi hinzufügen](new-client.md) - dort wird ein SD-Image automatisch gebaut.
+
+## Übersicht
+
+Bei der Erstinstallation gibt es ein Henne-Ei-Problem:
+- SOPS-Secrets werden mit dem SSH-Host-Key verschlüsselt
+- Der SSH-Host-Key wird erst bei der Installation generiert
+- Daher: Erst installieren, dann Secrets konfigurieren
+
+## Voraussetzungen
+
+- Bootbares NixOS ISO ([Minimal ISO](https://nixos.org/download/#nixos-iso))
+- Netzwerkverbindung
+- Host-Konfiguration in `hosts/<hostname>/` (ohne secrets.yaml)
+
+## Schritt 1: Host-Konfiguration vorbereiten
+
+### 1.1 Template kopieren
+
+```bash
+cp -r templates/generic-server hosts/neuer-server
+```
+
+### 1.2 Hostname setzen
+
+`hosts/neuer-server/networking.nix`:
+
+```nix
+{
+  networking.hostName = "neuer-server";
+}
+```
+
+### 1.3 In flake.nix registrieren
+
+```nix
+nixosConfigurations = {
+  neuer-server = mkNixosConfiguration "x86_64-linux" [ ./hosts/neuer-server ];
+};
+```
+
+### 1.4 Placeholder secrets.yaml erstellen
+
+```bash
+touch hosts/neuer-server/secrets.yaml
+```
+
+### 1.5 SOPS-Secrets temporär deaktivieren
+
+In `hosts/neuer-server/default.nix` alle `sops.secrets.*` Referenzen auskommentieren oder mit `lib.mkIf false` umgeben, bis die echten Secrets existieren.
+
+## Schritt 2: Zielmaschine vorbereiten
+
+### 2.1 NixOS ISO booten
+
+Von USB/CD booten.
+
+### 2.2 Root-Passwort setzen (für SSH)
+
+```bash
+passwd
+```
+
+### 2.3 IP-Adresse ermitteln
+
+```bash
+ip a
+```
+
+### 2.4 Per SSH verbinden (optional)
+
+```bash
+ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no nixos@<IP>
+sudo -i
+```
+
+## Schritt 3: Installation durchführen
+
+### 3.1 Repository klonen
+
+```bash
+nix-shell -p git
+git clone <GIT_REPO_URL> /tmp/nixos
+cd /tmp/nixos
+```
+
+### 3.2 Disk-Konfiguration anpassen
+
+**Wichtig:** Die Disk-ID muss zur Hardware passen!
+
+```bash
+# Verfügbare Disks anzeigen
+lsblk -o NAME,SIZE,MODEL,SERIAL
+ls -la /dev/disk/by-id/
+```
+
+In `hosts/neuer-server/disks.sh` oder `disks.nix` die richtige Disk-ID eintragen.
+
+### 3.3 Install-Script ausführen
+
+```bash
+bash scripts/install.sh -n neuer-server
+```
+
+Das Script:
+1. Partitioniert die Disk (via disko oder disks.sh)
+2. Generiert hardware.nix (falls nicht vorhanden)
+3. Installiert NixOS
+
+### 3.4 Reboot
+
+```bash
+umount -Rl /mnt
+reboot
+```
+
+## Schritt 4: Nach dem ersten Boot
+
+### 4.1 Einloggen
+
+Standard-Passwort: `changeme`
+
+```bash
+passwd  # Sofort ändern!
+```
+
+### 4.2 SSH-Host-Key zu Age-Key konvertieren
+
+```bash
+nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+```
+
+**Ausgabe notieren!** (z.B. `age1abc123...`)
+
+### 4.3 Auf Entwicklungsrechner: SOPS konfigurieren
+
+`.sops.yaml` bearbeiten:
+
+```yaml
+keys:
+  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
+  - &neuer_server_key age1abc123...  # Key von oben
+
+creation_rules:
+  - path_regex: hosts/neuer-server/secrets.yaml$
+    key_groups:
+      - age:
+        - *admin_key
+        - *neuer_server_key
+```
+
+### 4.4 Secrets erstellen
+
+```bash
+sops hosts/neuer-server/secrets.yaml
+```
+
+Mindestens den Tailscale Auth-Key eintragen (siehe nächster Schritt).
+
+### 4.5 SOPS-Referenzen wieder aktivieren
+
+Die in Schritt 1.5 auskommentierten `sops.secrets.*` Referenzen wieder aktivieren.
+
+### 4.6 Konfiguration deployen
+
+```bash
+# Lokal bauen und per SSH deployen
+nixos-rebuild switch --flake .#neuer-server --target-host root@<IP>
+```
+
+## Nächste Schritte
+
+- [Tailscale einrichten](../services/tailscale.md) - VPN-Verbindung
+- [Netdata konfigurieren](../services/netdata.md) - Monitoring
+- [CD einrichten](../deployment/cd.md) - Automatisches Deployment
--- a/docs/getting-started/new-client.md
+++ b/docs/getting-started/new-client.md
@ -0,0 +1,282 @@
+# Neuen Raspberry Pi Client hinzufügen
+
+Diese Anleitung beschreibt das Hinzufügen eines **neuen Raspberry Pi Clients** zur Infrastruktur.
+
+## Übersicht: Der Ablauf
+
+```
+1. Konfiguration erstellen     ──► Template kopieren, anpassen
+         │
+         ▼
+2. Zur Image-Pipeline hinzufügen ──► Workflow-Matrix erweitern
+         │
+         ▼
+3. Push auf main              ──► Forgejo baut automatisch SD-Image
+         │
+         ▼
+4. Image flashen & booten     ──► SD-Karte beschreiben, Pi starten
+         │
+         ▼
+5. SOPS konfigurieren         ──► Age-Key holen, Secrets erstellen
+         │
+         ▼
+6. Finales Deployment         ──► Tailscale etc. aktivieren
+```
+
+## Voraussetzungen
+
+- SSH-Zugang zu cryodev-main (für Tailscale Auth-Key)
+- Entwicklungsrechner mit Repository-Zugriff
+- SD-Karte (mindestens 8 GB)
+
+---
+
+## Schritt 1: Tailscale Auth-Key generieren
+
+**Auf cryodev-main** (per SSH):
+
+```bash
+sudo headscale preauthkeys create --expiration 99y --reusable --user default
+```
+
+**Ausgabe notieren!** (z.B. `tskey-preauth-abc123...`)
+
+---
+
+## Schritt 2: Host-Konfiguration erstellen
+
+### 2.1 Template kopieren
+
+```bash
+cp -r templates/raspberry-pi hosts/neuer-pi
+```
+
+### 2.2 Hostname setzen
+
+`hosts/neuer-pi/networking.nix`:
+
+```nix
+{
+  networking.hostName = "neuer-pi";
+}
+```
+
+### 2.3 In flake.nix registrieren
+
+```nix
+nixosConfigurations = {
+  # ... bestehende Hosts ...
+  
+  neuer-pi = mkNixosConfiguration "aarch64-linux" [ ./hosts/neuer-pi ];
+};
+```
+
+### 2.4 In constants.nix eintragen
+
+```nix
+{
+  hosts = {
+    # ... bestehende Hosts ...
+    
+    neuer-pi = {
+      ip = "100.64.0.X";  # Wird von Headscale vergeben
+    };
+  };
+}
+```
+
+### 2.5 Placeholder secrets.yaml erstellen
+
+```bash
+touch hosts/neuer-pi/secrets.yaml
+```
+
+### 2.6 SOPS temporär deaktivieren
+
+In `hosts/neuer-pi/default.nix` die `sops.secrets.*` Referenzen auskommentieren, damit das Image ohne Secrets gebaut werden kann.
+
+---
+
+## Schritt 3: Zur Image-Pipeline hinzufügen
+
+Bearbeite `.forgejo/workflows/build-pi-image.yml`:
+
+```yaml
+jobs:
+  build-pi-images:
+    strategy:
+      matrix:
+        # Neuen Host hier hinzufügen:
+        host: [cryodev-pi, neuer-pi]
+```
+
+---
+
+## Schritt 4: Push und Image bauen lassen
+
+```bash
+git add .
+git commit -m "Add neuer-pi host configuration"
+git push
+```
+
+Der Forgejo Workflow baut jetzt automatisch ein SD-Image für `neuer-pi`.
+
+**Warten** bis der Workflow fertig ist (30-60 Minuten). Status prüfen unter:
+`https://git.cryodev.xyz/steffen/cryodev-server/actions`
+
+---
+
+## Schritt 5: Image flashen
+
+### 5.1 Image herunterladen
+
+Nach erfolgreichem Build unter **Releases**:
+
+```bash
+wget https://git.cryodev.xyz/steffen/cryodev-server/releases/latest/download/neuer-pi-sd-image.img.zst
+```
+
+### 5.2 Dekomprimieren
+
+```bash
+zstd -d neuer-pi-sd-image.img.zst -o neuer-pi.img
+```
+
+### 5.3 Auf SD-Karte schreiben
+
+**Achtung:** `/dev/sdX` durch das richtige Gerät ersetzen!
+
+```bash
+lsblk  # Richtiges Gerät finden
+sudo dd if=neuer-pi.img of=/dev/sdX bs=4M conv=fsync status=progress
+```
+
+### 5.4 Booten
+
+1. SD-Karte in den Raspberry Pi einlegen
+2. Ethernet anschließen
+3. Strom anschließen
+4. Warten bis gebootet (ca. 2 Minuten)
+
+---
+
+## Schritt 6: SOPS konfigurieren
+
+### 6.1 IP-Adresse finden
+
+Der Pi sollte per DHCP eine IP bekommen. Prüfe deinen Router oder scanne das Netzwerk:
+
+```bash
+nmap -sn 192.168.1.0/24 | grep -B2 "Raspberry"
+```
+
+### 6.2 SSH verbinden
+
+```bash
+ssh steffen@<IP>  # oder der konfigurierte User
+```
+
+Standard-Passwort siehe `hosts/neuer-pi/users.nix`.
+
+### 6.3 Age-Key ermitteln
+
+Auf dem Pi:
+
+```bash
+nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+```
+
+**Ausgabe notieren!** (z.B. `age1xyz...`)
+
+### 6.4 .sops.yaml aktualisieren
+
+Auf dem Entwicklungsrechner:
+
+```yaml
+keys:
+  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
+  - &neuer_pi_key age1xyz...  # Der neue Key
+
+creation_rules:
+  # ... bestehende Regeln ...
+  
+  - path_regex: hosts/neuer-pi/secrets.yaml$
+    key_groups:
+      - age:
+        - *admin_key
+        - *neuer_pi_key
+```
+
+### 6.5 Secrets erstellen
+
+```bash
+sops hosts/neuer-pi/secrets.yaml
+```
+
+Inhalt:
+
+```yaml
+tailscale:
+  auth-key: "tskey-preauth-abc123..."  # Key aus Schritt 1
+
+netdata:
+  stream:
+    child-uuid: "..."  # uuidgen
+```
+
+### 6.6 SOPS-Referenzen aktivieren
+
+Die in Schritt 2.6 auskommentierten `sops.secrets.*` Referenzen wieder aktivieren.
+
+---
+
+## Schritt 7: Finales Deployment
+
+```bash
+git add .
+git commit -m "Configure SOPS secrets for neuer-pi"
+git push
+```
+
+Da Comin auf dem Pi läuft, wird er die neue Konfiguration automatisch pullen.
+
+Alternativ manuell:
+
+```bash
+nixos-rebuild switch --flake .#neuer-pi --target-host root@<IP>
+```
+
+---
+
+## Schritt 8: Verifizieren
+
+### Tailscale-Verbindung
+
+```bash
+# Auf dem Pi
+tailscale status
+
+# Auf cryodev-main
+sudo headscale nodes list
+```
+
+### Netdata-Streaming
+
+Prüfe ob der neue Client im Netdata-Dashboard erscheint:
+`https://netdata.cryodev.xyz`
+
+---
+
+## Checkliste
+
+- [ ] Tailscale Auth-Key auf cryodev-main generiert
+- [ ] Host-Konfiguration erstellt (Template, flake.nix, constants.nix)
+- [ ] Host zur Workflow-Matrix hinzugefügt
+- [ ] Gepusht und auf Image-Build gewartet
+- [ ] SD-Karte geflasht und Pi gebootet
+- [ ] Age-Key ermittelt und in .sops.yaml eingetragen
+- [ ] secrets.yaml erstellt (Tailscale-Key, Netdata-UUID)
+- [ ] SOPS-Referenzen aktiviert und deployed
+- [ ] Tailscale-Verbindung funktioniert
+- [ ] Netdata-Streaming funktioniert
--- a/docs/getting-started/prerequisites.md
+++ b/docs/getting-started/prerequisites.md
@ -0,0 +1,63 @@
+# Prerequisites
+
+## Required Tools
+
+Ensure you have the following tools installed on your local machine:
+
+| Tool | Purpose |
+|------|---------|
+| `nix` | Package manager with flakes enabled |
+| `sops` | Secret encryption/decryption |
+| `age` | Encryption backend for sops |
+| `ssh` | Remote access |
+
+### Installing Nix
+
+Follow the [official Nix installation guide](https://nixos.org/download/).
+
+Enable flakes by adding to `~/.config/nix/nix.conf`:
+
+```
+experimental-features = nix-command flakes
+```
+
+### Installing Other Tools
+
+With Nix:
+
+```bash
+nix-shell -p sops age
+```
+
+Or install globally via home-manager or system configuration.
+
+## Repository Access
+
+Clone the repository:
+
+```bash
+git clone https://git.cryodev.xyz/steffen/cryodev-server.git
+cd cryodev-server
+```
+
+## Development Shell
+
+Enter the development shell with all required tools:
+
+```bash
+nix develop
+```
+
+## Verifying Setup
+
+Check that the flake is valid:
+
+```bash
+nix flake check
+```
+
+Build a host configuration (dry run):
+
+```bash
+nix build .#nixosConfigurations.cryodev-main.config.system.build.toplevel --dry-run
+```
--- a/docs/getting-started/reinstall.md
+++ b/docs/getting-started/reinstall.md
@ -0,0 +1,183 @@
+# Neuinstallation (Reinstall)
+
+Diese Anleitung beschreibt die **Neuinstallation** eines bestehenden Hosts, z.B. nach Hardwarewechsel oder bei Problemen.
+
+## Unterschied zur Erstinstallation
+
+| Aspekt | Erstinstallation | Neuinstallation |
+|--------|------------------|-----------------|
+| SOPS-Secrets | Noch nicht vorhanden | Bereits konfiguriert |
+| SSH-Host-Key | Neu generiert | **Muss wiederhergestellt werden!** |
+| Disk-IDs | Neu ermitteln | Oft geändert (neue Hardware) |
+| secrets.yaml | Wird erstellt | Bereits vorhanden |
+
+## Wichtig: SSH-Host-Key Problem
+
+Bei einer Neuinstallation wird ein **neuer SSH-Host-Key** generiert. Dieser stimmt nicht mehr mit dem Age-Key in `.sops.yaml` überein!
+
+### Lösungsmöglichkeiten
+
+**Option A: Alten Host-Key sichern und wiederherstellen** (empfohlen)
+
+**Option B: Neuen Key generieren und SOPS aktualisieren**
+
+## Voraussetzungen
+
+- Backup des alten SSH-Host-Keys (falls Option A)
+- Zugriff auf `.sops.yaml` und die Admin-Age-Keys
+- Bootbares NixOS ISO
+
+## Schritt 1: Vorbereitung (vor der Installation)
+
+### 1.1 Alten SSH-Host-Key sichern (Option A)
+
+Falls der alte Host noch läuft:
+
+```bash
+# Auf dem alten Host
+sudo cat /etc/ssh/ssh_host_ed25519_key > ~/ssh_host_ed25519_key.backup
+sudo cat /etc/ssh/ssh_host_ed25519_key.pub > ~/ssh_host_ed25519_key.pub.backup
+```
+
+Dateien sicher auf den Entwicklungsrechner kopieren.
+
+### 1.2 Disk-IDs ermitteln
+
+**Bei neuer Hardware** ändern sich die Disk-IDs!
+
+```bash
+# Im NixOS Live-System
+lsblk -o NAME,SIZE,MODEL,SERIAL
+ls -la /dev/disk/by-id/
+```
+
+Die neue Disk-ID in `hosts/<hostname>/disks.sh` oder `disks.nix` eintragen:
+
+```bash
+# Beispiel disks.sh
+DISK="/dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_XXXXX"
+```
+
+## Schritt 2: Installation durchführen
+
+### 2.1 NixOS ISO booten
+
+Von USB/CD booten, Root-Passwort setzen, per SSH verbinden.
+
+### 2.2 Repository klonen
+
+```bash
+sudo -i
+nix-shell -p git
+git clone <GIT_REPO_URL> /tmp/nixos
+cd /tmp/nixos
+```
+
+### 2.3 Disk-Konfiguration prüfen
+
+```bash
+# Aktuelle Disk-IDs anzeigen
+ls -la /dev/disk/by-id/
+
+# Mit Konfiguration vergleichen
+cat hosts/<hostname>/disks.sh | grep DISK
+```
+
+**Falls nötig:** Disk-ID in der Konfiguration anpassen.
+
+### 2.4 Install-Script ausführen
+
+```bash
+bash scripts/install.sh -n <hostname>
+```
+
+### 2.5 SSH-Host-Key wiederherstellen (Option A)
+
+**Vor dem Reboot!**
+
+```bash
+# Host-Key vom Backup wiederherstellen
+cp /path/to/ssh_host_ed25519_key.backup /mnt/etc/ssh/ssh_host_ed25519_key
+cp /path/to/ssh_host_ed25519_key.pub.backup /mnt/etc/ssh/ssh_host_ed25519_key.pub
+chmod 600 /mnt/etc/ssh/ssh_host_ed25519_key
+chmod 644 /mnt/etc/ssh/ssh_host_ed25519_key.pub
+```
+
+### 2.6 Reboot
+
+```bash
+umount -Rl /mnt
+reboot
+```
+
+## Schritt 3: Nach dem Reboot
+
+### Bei Option A (Key wiederhergestellt)
+
+SOPS-Secrets sollten automatisch funktionieren. Testen:
+
+```bash
+sudo cat /run/secrets/tailscale/auth-key
+```
+
+### Bei Option B (Neuer Key)
+
+Der Host kann die Secrets nicht entschlüsseln. Neuen Key konfigurieren:
+
+```bash
+# Neuen Age-Key ermitteln
+nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+```
+
+Auf dem Entwicklungsrechner:
+
+```bash
+# .sops.yaml aktualisieren mit neuem Key
+vim .sops.yaml
+
+# Secrets mit neuem Key neu verschlüsseln
+sops updatekeys hosts/<hostname>/secrets.yaml
+```
+
+Dann Konfiguration neu deployen:
+
+```bash
+nixos-rebuild switch --flake .#<hostname> --target-host root@<IP>
+```
+
+## Häufige Probleme
+
+### "No secret key available"
+
+SOPS kann die Secrets nicht entschlüsseln. Ursache:
+- SSH-Host-Key stimmt nicht mit Age-Key in `.sops.yaml` überein
+
+Lösung: Option B durchführen (neuen Key konfigurieren).
+
+### "Device not found" beim Partitionieren
+
+Disk-ID in `disks.sh`/`disks.nix` ist falsch.
+
+```bash
+# Richtige ID finden
+ls -la /dev/disk/by-id/
+```
+
+### Hardware-Config veraltet
+
+Bei neuer Hardware muss `hardware.nix` neu generiert werden:
+
+```bash
+# Install-Script generiert automatisch neu, falls Datei fehlt
+rm hosts/<hostname>/hardware.nix
+bash scripts/install.sh -n <hostname>
+```
+
+## Checkliste
+
+- [ ] Alten SSH-Host-Key gesichert (falls möglich)
+- [ ] Disk-IDs in Konfiguration geprüft/aktualisiert
+- [ ] Installation durchgeführt
+- [ ] SSH-Host-Key wiederhergestellt ODER neuen Key in SOPS konfiguriert
+- [ ] Secrets funktionieren (`sudo cat /run/secrets/...`)
+- [ ] Tailscale verbunden (`tailscale status`)
--- a/docs/getting-started/sd-image.md
+++ b/docs/getting-started/sd-image.md
@ -0,0 +1,116 @@
+# SD-Karten-Images für Raspberry Pi
+
+Das Repository baut automatisch SD-Karten-Images für alle konfigurierten Raspberry Pi Hosts.
+
+## Automatischer Build
+
+Bei Änderungen an `main` werden automatisch Images für alle Pi-Hosts gebaut und als Release veröffentlicht.
+
+**Download:** [Releases auf Forgejo](https://git.cryodev.xyz/steffen/cryodev-server/releases)
+
+## Verfügbare Images
+
+| Host | Image-Name |
+|------|------------|
+| `cryodev-pi` | `cryodev-pi-sd-image.img.zst` |
+
+Neue Hosts werden automatisch gebaut, wenn sie zur Workflow-Matrix hinzugefügt werden.
+
+## Image flashen
+
+### 1. Herunterladen
+
+```bash
+wget https://git.cryodev.xyz/.../releases/latest/download/<hostname>-sd-image.img.zst
+wget https://git.cryodev.xyz/.../releases/latest/download/<hostname>-sd-image.img.zst.sha256
+
+# Checksum prüfen
+sha256sum -c <hostname>-sd-image.img.zst.sha256
+```
+
+### 2. Dekomprimieren
+
+```bash
+zstd -d <hostname>-sd-image.img.zst -o <hostname>.img
+```
+
+### 3. Auf SD-Karte schreiben
+
+```bash
+# Richtiges Gerät finden
+lsblk
+
+# Schreiben (ACHTUNG: richtiges Gerät wählen!)
+sudo dd if=<hostname>.img of=/dev/sdX bs=4M conv=fsync status=progress
+```
+
+Alternativ: `balenaEtcher` oder `Raspberry Pi Imager` verwenden.
+
+## Was ist im Image?
+
+- Vollständige NixOS-Installation für den spezifischen Host
+- Alle konfigurierten Services (außer Secrets)
+- SSH-Server aktiviert
+- Automatische Root-Partition-Erweiterung beim ersten Boot
+- Comin für automatische Updates
+
+## Was fehlt?
+
+**SOPS-Secrets** können nicht im Image enthalten sein (Henne-Ei-Problem mit SSH-Host-Key).
+
+Nach dem ersten Boot:
+1. Age-Key vom Pi holen
+2. `.sops.yaml` aktualisieren
+3. `secrets.yaml` erstellen
+4. Konfiguration deployen
+
+Siehe [Neuen Client hinzufügen](new-client.md) für die vollständige Anleitung.
+
+## Neuen Host zur Pipeline hinzufügen
+
+1. Host-Konfiguration in `hosts/<hostname>/` erstellen
+2. In `.forgejo/workflows/build-pi-image.yml` zur Matrix hinzufügen:
+
+```yaml
+matrix:
+  host: [cryodev-pi, neuer-host]  # <- hier hinzufügen
+```
+
+3. Push auf `main` → Image wird automatisch gebaut
+
+## Manuell bauen
+
+```bash
+# Auf aarch64 (z.B. anderem Pi)
+nix build .#nixosConfigurations.<hostname>.config.system.build.sdImage
+
+# Auf x86_64 mit QEMU-Emulation (langsam)
+nix build .#nixosConfigurations.<hostname>.config.system.build.sdImage \
+  --extra-platforms aarch64-linux
+```
+
+Voraussetzung auf x86_64:
+
+```nix
+{
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
+}
+```
+
+## Troubleshooting
+
+### Workflow schlägt fehl
+
+- Prüfe ob `sd-image.nix` in der Host-Konfiguration importiert wird
+- Prüfe ob binfmt auf cryodev-main aktiviert ist
+
+### Image bootet nicht
+
+- SD-Karte korrekt beschrieben?
+- Andere SD-Karte versuchen
+- Stromversorgung prüfen (min. 3A für Pi 4)
+
+### Kein Netzwerk
+
+- Ethernet-Kabel prüfen
+- DHCP-Server im Netzwerk?
--- a/docs/index.md
+++ b/docs/index.md
@ -0,0 +1,94 @@
+# Cryodev NixOS Configuration Documentation
+
+Willkommen zur Dokumentation der **cryodev** NixOS-Infrastruktur.
+
+## Quick Links
+
+### Getting Started
+
+- [Voraussetzungen](getting-started/prerequisites.md) - Benötigte Tools
+- [Neuen Raspberry Pi hinzufügen](getting-started/new-client.md) - Kompletter Workflow für neue Clients
+- [SD-Image Referenz](getting-started/sd-image.md) - Details zum Image-Build
+- [Erstinstallation (Server)](getting-started/first-install.md) - Bootstrap für x86_64 Hosts
+- [Neuinstallation](getting-started/reinstall.md) - Reinstall mit Hardware-Änderungen
+
+### Services
+
+- [SOPS Secrets](services/sops.md) - Geheimnisverwaltung mit sops-nix
+- [Headscale](services/headscale.md) - Self-hosted Tailscale Server
+- [Headplane](services/headplane.md) - Web-UI für Headscale
+- [Tailscale](services/tailscale.md) - Mesh-VPN Client
+- [Mailserver](services/mailserver.md) - E-Mail Stack (Postfix/Dovecot)
+- [Forgejo](services/forgejo.md) - Git-Hosting mit CI/CD
+- [Netdata](services/netdata.md) - Monitoring und Alerting
+
+### Deployment
+
+- [Continuous Deployment](deployment/cd.md) - Push- und Pull-basiertes Deployment
+- [DNS-Konfiguration](deployment/dns.md) - Benötigte DNS-Einträge
+
+## Architektur
+
+```
+                    Internet
+                        |
+                   cryodev.xyz
+                        |
+              +-------------------+
+              |   cryodev-main    |
+              |   (x86_64 Server) |
+              +-------------------+
+              | - Headscale       |
+              | - Headplane       |
+              | - Forgejo         |
+              | - Mailserver      |
+              | - Netdata Parent  |
+              +-------------------+
+                        |
+               Tailscale Mesh VPN
+                        |
+              +-------------------+
+              |   cryodev-pi      |
+              | (Raspberry Pi 4)  |
+              +-------------------+
+              | - Tailscale       |
+              | - Netdata Child   |
+              | - Comin (GitOps)  |
+              +-------------------+
+```
+
+## Installations-Szenarien
+
+| Szenario | Beschreibung | Anleitung |
+|----------|--------------|-----------|
+| **Neuer Raspberry Pi** | Config erstellen → Image bauen → Flashen | [new-client.md](getting-started/new-client.md) |
+| **Erstinstallation (Server)** | x86_64 Host, manuelle Installation | [first-install.md](getting-started/first-install.md) |
+| **Neuinstallation** | Bestehender Host, neue Hardware | [reinstall.md](getting-started/reinstall.md) |
+
+Für Raspberry Pi: [SD-Image Referenz](getting-started/sd-image.md)
+
+## Verzeichnisstruktur
+
+```
+.
+├── flake.nix           # Entry point, inputs and outputs
+├── constants.nix       # Zentrale Config (Domains, IPs, Ports)
+├── hosts/              # Host-spezifische Konfigurationen
+│   ├── cryodev-main/
+│   └── cryodev-pi/
+├── modules/            # Wiederverwendbare NixOS-Module
+│   └── nixos/
+├── pkgs/               # Eigene Pakete
+├── overlays/           # Nixpkgs Overlays
+├── templates/          # Templates für neue Hosts
+├── scripts/            # Helper-Scripts (install.sh)
+├── apps/               # Nix Apps (rebuild)
+└── lib/                # Helper-Funktionen (utils.nix)
+```
+
+## Deployment-Strategien
+
+| Host | Strategie | Tool | Beschreibung |
+|------|-----------|------|--------------|
+| `cryodev-main` | Push-basiert | deploy-rs via Forgejo Actions | Sofortige Updates bei Push |
+| `cryodev-pi` | Pull-basiert | Comin | Pollt Repository auf Änderungen |
--- a/docs/services/forgejo.md
+++ b/docs/services/forgejo.md
@ -0,0 +1,149 @@
+# Forgejo
+
+Forgejo is a self-hosted Git service (fork of Gitea) with built-in CI/CD Actions.
+
+## References
+
+- [Forgejo Documentation](https://forgejo.org/docs/)
+- [Forgejo Actions](https://forgejo.org/docs/latest/user/actions/)
+
+## Setup
+
+### DNS
+
+Set a CNAME record for `git.cryodev.xyz` pointing to your main domain.
+
+### Configuration
+
+```nix
+# hosts/cryodev-main/services/forgejo.nix
+{ config, ... }:
+{
+  services.forgejo = {
+    enable = true;
+    settings = {
+      server = {
+        DOMAIN = "git.cryodev.xyz";
+        ROOT_URL = "https://git.cryodev.xyz";
+      };
+      mailer = {
+        ENABLED = true;
+        FROM = "forgejo@cryodev.xyz";
+      };
+    };
+  };
+}
+```
+
+## Forgejo Runner
+
+The runner executes CI/CD pipelines defined in `.forgejo/workflows/`.
+
+### Get Runner Token
+
+1. Go to Forgejo Admin Panel
+2. Navigate to Actions > Runners
+3. Create a new runner and copy the token
+
+### Add to Secrets
+
+```bash
+sops hosts/cryodev-main/secrets.yaml
+```
+
+```yaml
+forgejo-runner:
+  token: "your-runner-token"
+```
+
+### Configuration
+
+```nix
+{
+  sops.secrets."forgejo-runner/token" = { };
+  
+  services.gitea-actions-runner = {
+    instances.default = {
+      enable = true;
+      url = "https://git.cryodev.xyz";
+      tokenFile = config.sops.secrets."forgejo-runner/token".path;
+      labels = [ "ubuntu-latest:docker://node:20" ];
+    };
+  };
+}
+```
+
+## CI/CD Workflows
+
+### deploy-rs Workflow
+
+`.forgejo/workflows/deploy.yaml`:
+
+```yaml
+name: Deploy
+on:
+  push:
+    branches: [main]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Install Nix
+        uses: cachix/install-nix-action@v24
+        
+      - name: Deploy
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          nix run .#deploy
+```
+
+## Administration
+
+### Create Admin User
+
+```bash
+sudo -u forgejo forgejo admin user create \
+  --username admin \
+  --password changeme \
+  --email admin@cryodev.xyz \
+  --admin
+```
+
+### Reset User Password
+
+```bash
+sudo -u forgejo forgejo admin user change-password \
+  --username USER \
+  --password NEWPASS
+```
+
+## Troubleshooting
+
+### Check Service Status
+
+```bash
+sudo systemctl status forgejo
+sudo systemctl status gitea-runner-default
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u forgejo -f
+sudo journalctl -u gitea-runner-default -f
+```
+
+### Database Issues
+
+Forgejo uses SQLite by default. Database location:
+
+```bash
+ls -la /var/lib/forgejo/data/
+```
--- a/docs/services/headplane.md
+++ b/docs/services/headplane.md
@ -0,0 +1,107 @@
+# Headplane
+
+Headplane is a web-based admin interface for Headscale.
+
+## References
+
+- [GitHub](https://github.com/tale/headplane)
+
+## Setup
+
+### DNS
+
+Set a CNAME record for `headplane.cryodev.xyz` pointing to your main domain.
+
+### Generate Secrets
+
+**Cookie Secret** (for session management):
+
+```bash
+nix-shell -p openssl --run 'openssl rand -hex 16'
+```
+
+**Agent Pre-Auth Key** (for Headplane's built-in agent):
+
+```bash
+# First, create a dedicated user
+sudo headscale users create headplane-agent
+
+# Then create a reusable pre-auth key
+sudo headscale preauthkeys create --expiration 99y --reusable --user headplane-agent
+```
+
+### Add to Secrets
+
+Edit `hosts/cryodev-main/secrets.yaml`:
+
+```bash
+sops hosts/cryodev-main/secrets.yaml
+```
+
+```yaml
+headplane:
+  cookie_secret: "your-generated-hex-string"
+  agent_pre_authkey: "your-preauth-key"
+```
+
+### Configuration
+
+```nix
+# hosts/cryodev-main/services/headplane.nix
+{ config, ... }:
+{
+  sops.secrets."headplane/cookie_secret" = { };
+  sops.secrets."headplane/agent_pre_authkey" = { };
+  
+  services.headplane = {
+    enable = true;
+    settings = {
+      server = {
+        cookie_secret_file = config.sops.secrets."headplane/cookie_secret".path;
+      };
+      headscale = {
+        url = "https://headscale.cryodev.xyz";
+      };
+      agent = {
+        enable = true;
+        authkey_file = config.sops.secrets."headplane/agent_pre_authkey".path;
+      };
+    };
+  };
+}
+```
+
+## Usage
+
+Access Headplane at `https://headplane.cryodev.xyz`.
+
+### Features
+
+- View and manage users
+- View connected nodes
+- Manage routes and exit nodes
+- View pre-auth keys
+
+## Troubleshooting
+
+### Check Service Status
+
+```bash
+sudo systemctl status headplane
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u headplane -f
+```
+
+### Agent Not Connecting
+
+Verify the agent pre-auth key is valid:
+
+```bash
+sudo headscale preauthkeys list --user headplane-agent
+```
+
+If expired, create a new one and update the secrets file.
--- a/docs/services/headscale.md
+++ b/docs/services/headscale.md
@ -0,0 +1,116 @@
+# Headscale
+
+Headscale is an open-source, self-hosted implementation of the Tailscale control server.
+
+## References
+
+- [Website](https://headscale.net/stable/)
+- [GitHub](https://github.com/juanfont/headscale)
+- [Example configuration](https://github.com/juanfont/headscale/blob/main/config-example.yaml)
+
+## Setup
+
+### DNS
+
+Set a CNAME record for `headscale.cryodev.xyz` pointing to your main domain.
+
+### Configuration
+
+```nix
+# hosts/cryodev-main/services/headscale.nix
+{
+  services.headscale = {
+    enable = true;
+    openFirewall = true;
+  };
+}
+```
+
+## Usage
+
+### Create a User
+
+```bash
+sudo headscale users create <USERNAME>
+```
+
+### List Users
+
+```bash
+sudo headscale users list
+```
+
+### Create Pre-Auth Key
+
+```bash
+sudo headscale preauthkeys create --expiration 99y --reusable --user <USER_ID>
+```
+
+The pre-auth key is used by clients to automatically authenticate and join the tailnet.
+
+### List Nodes
+
+```bash
+sudo headscale nodes list
+```
+
+### Delete a Node
+
+```bash
+sudo headscale nodes delete -i <NODE_ID>
+```
+
+### Rename a Node
+
+```bash
+sudo headscale nodes rename -i <NODE_ID> new-name
+```
+
+## ACL Configuration
+
+Access Control Lists define which nodes can communicate with each other.
+
+### Validate ACL File
+
+```bash
+sudo headscale policy check --file /path/to/acl.hujson
+```
+
+### Example ACL
+
+```json
+{
+  "acls": [
+    {
+      "action": "accept",
+      "src": ["*"],
+      "dst": ["*:*"]
+    }
+  ]
+}
+```
+
+## Troubleshooting
+
+### Check Service Status
+
+```bash
+sudo systemctl status headscale
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u headscale -f
+```
+
+### Test DERP Connectivity
+
+```bash
+curl -I https://headscale.cryodev.xyz/derp
+```
+
+## Integration
+
+- [Headplane](headplane.md) - Web UI for managing Headscale
+- [Tailscale Client](tailscale.md) - Connect clients to Headscale
--- a/docs/services/mailserver.md
+++ b/docs/services/mailserver.md
@ -0,0 +1,147 @@
+# Mailserver
+
+NixOS mailserver module providing a complete email stack with Postfix and Dovecot.
+
+## References
+
+- [Simple NixOS Mailserver](https://gitlab.com/simple-nixos-mailserver/nixos-mailserver)
+
+## Setup
+
+### DNS Records
+
+| Type | Hostname | Value |
+|------|----------|-------|
+| A | `mail` | `<SERVER_IP>` |
+| AAAA | `mail` | `<SERVER_IPV6>` |
+| MX | `@` | `10 mail.cryodev.xyz.` |
+| TXT | `@` | `"v=spf1 mx ~all"` |
+| TXT | `_dmarc` | `"v=DMARC1; p=none"` |
+
+DKIM records are generated automatically after first deployment.
+
+### Generate Password Hashes
+
+```bash
+nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'
+```
+
+### Add to Secrets
+
+```bash
+sops hosts/cryodev-main/secrets.yaml
+```
+
+```yaml
+mailserver:
+  accounts:
+    admin: "$2y$05$..."
+    forgejo: "$2y$05$..."
+```
+
+### Configuration
+
+```nix
+# hosts/cryodev-main/services/mailserver.nix
+{ config, ... }:
+{
+  sops.secrets."mailserver/accounts/admin" = { };
+  sops.secrets."mailserver/accounts/forgejo" = { };
+  
+  mailserver = {
+    enable = true;
+    fqdn = "mail.cryodev.xyz";
+    domains = [ "cryodev.xyz" ];
+    
+    loginAccounts = {
+      "admin@cryodev.xyz" = {
+        hashedPasswordFile = config.sops.secrets."mailserver/accounts/admin".path;
+      };
+      "forgejo@cryodev.xyz" = {
+        hashedPasswordFile = config.sops.secrets."mailserver/accounts/forgejo".path;
+        sendOnly = true;
+      };
+    };
+  };
+}
+```
+
+## DKIM Setup
+
+After first deployment, get the DKIM public key:
+
+```bash
+sudo cat /var/dkim/cryodev.xyz.mail.txt
+```
+
+Add this as a TXT record:
+
+| Type | Hostname | Value |
+|------|----------|-------|
+| TXT | `mail._domainkey` | `v=DKIM1; k=rsa; p=...` |
+
+## Testing
+
+### Send Test Email
+
+```bash
+echo "Test" | mail -s "Test Subject" recipient@example.com
+```
+
+### Check Mail Queue
+
+```bash
+sudo postqueue -p
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u postfix -f
+sudo journalctl -u dovecot2 -f
+```
+
+### Test SMTP
+
+```bash
+openssl s_client -connect mail.cryodev.xyz:587 -starttls smtp
+```
+
+### Verify DNS Records
+
+- [MXToolbox](https://mxtoolbox.com/)
+- [Mail-tester](https://www.mail-tester.com/)
+
+## Troubleshooting
+
+### Emails Not Sending
+
+Check Postfix status:
+
+```bash
+sudo systemctl status postfix
+```
+
+Check firewall (ports 25, 465, 587 must be open):
+
+```bash
+sudo iptables -L -n | grep -E '25|465|587'
+```
+
+### DKIM Failing
+
+Verify the DNS record matches the generated key:
+
+```bash
+dig TXT mail._domainkey.cryodev.xyz
+```
+
+### SPF Failing
+
+Verify SPF record:
+
+```bash
+dig TXT cryodev.xyz
+```
+
+Should return: `"v=spf1 mx ~all"`
--- a/docs/services/netdata.md
+++ b/docs/services/netdata.md
@ -0,0 +1,181 @@
+# Netdata Monitoring
+
+Netdata provides real-time performance monitoring with parent/child streaming.
+
+## Architecture
+
+```
+┌─────────────────┐     Stream over      ┌─────────────────┐
+│  cryodev-pi     │  ───────────────────>│  cryodev-main   │
+│  (Child Node)   │     Tailscale VPN    │  (Parent Node)  │
+└─────────────────┘                      └─────────────────┘
+                                                  │
+                                                  v
+                                         https://netdata.cryodev.xyz
+```
+
+## References
+
+- [Netdata Documentation](https://learn.netdata.cloud/)
+- [Streaming Configuration](https://learn.netdata.cloud/docs/streaming/streaming-configuration-reference)
+
+## Parent Node (cryodev-main)
+
+### DNS
+
+Set a CNAME record for `netdata.cryodev.xyz` pointing to your main domain.
+
+### Generate Stream API Key
+
+```bash
+uuidgen
+```
+
+### Configuration
+
+```nix
+# hosts/cryodev-main/services/netdata.nix
+{ config, ... }:
+{
+  sops.secrets."netdata/stream-api-key" = { };
+  
+  sops.templates."netdata-stream.conf" = {
+    content = ''
+      [${config.sops.placeholder."netdata/stream-api-key"}]
+      enabled = yes
+      default history = 3600
+      default memory mode = ram
+      health enabled by default = auto
+      allow from = *
+    '';
+    owner = "netdata";
+  };
+  
+  services.netdata = {
+    enable = true;
+    configDir."stream.conf" = config.sops.templates."netdata-stream.conf".path;
+  };
+}
+```
+
+## Child Node (cryodev-pi)
+
+### Generate Child UUID
+
+```bash
+uuidgen
+```
+
+### Add to Secrets
+
+```bash
+sops hosts/cryodev-pi/secrets.yaml
+```
+
+```yaml
+netdata:
+  stream:
+    child-uuid: "your-generated-uuid"
+```
+
+Note: The stream API key must match the parent's key. You can either:
+1. Share the same secret between hosts (complex with SOPS)
+2. Hardcode a known API key in both configurations
+
+### Configuration
+
+```nix
+# hosts/cryodev-pi/services/netdata.nix
+{ config, constants, ... }:
+{
+  sops.secrets."netdata/stream/child-uuid" = { };
+  
+  sops.templates."netdata-stream.conf" = {
+    content = ''
+      [stream]
+      enabled = yes
+      destination = ${constants.hosts.cryodev-main.ip}:19999
+      api key = YOUR_STREAM_API_KEY
+      send charts matching = *
+    '';
+    owner = "netdata";
+  };
+  
+  services.netdata = {
+    enable = true;
+    configDir."stream.conf" = config.sops.templates."netdata-stream.conf".path;
+  };
+}
+```
+
+## Email Alerts
+
+Configure Netdata to send alerts via the mailserver:
+
+```nix
+{
+  services.netdata.configDir."health_alarm_notify.conf" = pkgs.writeText "notify.conf" ''
+    SEND_EMAIL="YES"
+    EMAIL_SENDER="netdata@cryodev.xyz"
+    DEFAULT_RECIPIENT_EMAIL="admin@cryodev.xyz"
+  '';
+}
+```
+
+## Usage
+
+### Access Dashboard
+
+Open `https://netdata.cryodev.xyz` in your browser.
+
+### View Child Nodes
+
+Child nodes appear in the left sidebar under "Nodes".
+
+### Check Streaming Status
+
+On parent:
+```bash
+curl -s http://localhost:19999/api/v1/info | jq '.hosts'
+```
+
+On child:
+```bash
+curl -s http://localhost:19999/api/v1/info | jq '.streaming'
+```
+
+## Troubleshooting
+
+### Check Service Status
+
+```bash
+sudo systemctl status netdata
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u netdata -f
+```
+
+### Child Not Streaming
+
+1. Verify network connectivity:
+   ```bash
+   tailscale ping cryodev-main
+   nc -zv <parent-ip> 19999
+   ```
+
+2. Check API key matches between parent and child
+
+3. Verify firewall allows port 19999 on parent
+
+### High Memory Usage
+
+Adjust history settings in `netdata.conf`:
+
+```ini
+[global]
+history = 1800  # seconds to retain
+memory mode = ram
+```
--- a/docs/services/sops.md
+++ b/docs/services/sops.md
@ -0,0 +1,174 @@
+# SOPS Secret Management
+
+Atomic secret provisioning for NixOS using [sops-nix](https://github.com/Mic92/sops-nix).
+
+## Overview
+
+Secrets are encrypted with `age` using SSH host keys, ensuring:
+- No plaintext secrets in the repository
+- Secrets are decrypted at activation time
+- Each host can only decrypt its own secrets
+
+## Setup
+
+### 1. Get Host's Age Public Key
+
+After a host is installed, extract its age key from the SSH host key:
+
+```bash
+nix-shell -p ssh-to-age --run 'ssh-keyscan -t ed25519 <HOST_IP> | ssh-to-age'
+```
+
+Or locally on the host:
+
+```bash
+nix-shell -p ssh-to-age --run 'cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age'
+```
+
+### 2. Configure .sops.yaml
+
+Add the host key to `.sops.yaml`:
+
+```yaml
+keys:
+  - &admin_key age1e8p35795htf7twrejyugpzw0qja2v33awcw76y4gp6acnxnkzq0s935t4t
+  - &main_key age1...  # cryodev-main
+  - &pi_key age1...    # cryodev-pi
+
+creation_rules:
+  - path_regex: hosts/cryodev-main/secrets.yaml$
+    key_groups:
+      - age:
+        - *admin_key
+        - *main_key
+        
+  - path_regex: hosts/cryodev-pi/secrets.yaml$
+    key_groups:
+      - age:
+        - *admin_key
+        - *pi_key
+```
+
+### 3. Create Secrets File
+
+```bash
+sops hosts/<hostname>/secrets.yaml
+```
+
+This opens your editor. Add secrets in YAML format:
+
+```yaml
+tailscale:
+  auth-key: "tskey-..."
+  
+some-service:
+  password: "secret123"
+```
+
+## Usage in Modules
+
+### Declaring Secrets
+
+```nix
+{ config, ... }:
+{
+  sops.secrets.my-secret = {
+    # Optional: set owner/group
+    owner = "myservice";
+    group = "myservice";
+  };
+}
+```
+
+### Using Secrets
+
+Reference the secret path in service configuration:
+
+```nix
+{
+  services.myservice = {
+    passwordFile = config.sops.secrets.my-secret.path;
+  };
+}
+```
+
+### Using Templates
+
+For secrets that need to be embedded in config files:
+
+```nix
+{
+  sops.secrets."netdata/stream-api-key" = { };
+  
+  sops.templates."netdata-stream.conf" = {
+    content = ''
+      [stream]
+      enabled = yes
+      api key = ${config.sops.placeholder."netdata/stream-api-key"}
+    '';
+    owner = "netdata";
+  };
+  
+  services.netdata.configDir."stream.conf" = 
+    config.sops.templates."netdata-stream.conf".path;
+}
+```
+
+## Common Secrets
+
+### cryodev-main
+
+```yaml
+mailserver:
+  accounts:
+    forgejo: "$2y$05$..."  # bcrypt hash
+    admin: "$2y$05$..."
+
+forgejo-runner:
+  token: "..."
+
+headplane:
+  cookie_secret: "..."       # openssl rand -hex 16
+  agent_pre_authkey: "..."   # headscale preauthkey
+
+tailscale:
+  auth-key: "tskey-..."
+```
+
+### cryodev-pi
+
+```yaml
+tailscale:
+  auth-key: "tskey-..."
+
+netdata:
+  stream:
+    child-uuid: "..."  # uuidgen
+```
+
+## Generating Secret Values
+
+| Secret | Command |
+|--------|---------|
+| Mailserver password | `nix-shell -p mkpasswd --run 'mkpasswd -sm bcrypt'` |
+| Random hex token | `nix-shell -p openssl --run 'openssl rand -hex 16'` |
+| UUID | `uuidgen` |
+| Tailscale preauth | `sudo headscale preauthkeys create --expiration 99y --reusable --user default` |
+
+## Updating Keys
+
+After modifying `.sops.yaml`, update existing secrets files:
+
+```bash
+sops --config .sops.yaml updatekeys hosts/<hostname>/secrets.yaml
+```
+
+## Troubleshooting
+
+### "No matching keys found"
+
+Ensure the host's age key is in `.sops.yaml` and you've run `updatekeys`.
+
+### Secret not decrypting on host
+
+Check that `/etc/ssh/ssh_host_ed25519_key` exists and matches the public key in `.sops.yaml`.
--- a/docs/services/tailscale.md
+++ b/docs/services/tailscale.md
@ -0,0 +1,117 @@
+# Tailscale Client
+
+Tailscale clients connect to the self-hosted Headscale server to join the mesh VPN.
+
+## References
+
+- [Tailscale Documentation](https://tailscale.com/kb)
+- [Headscale Client Setup](https://headscale.net/running-headscale-linux/)
+
+## Setup
+
+### Generate Auth Key
+
+On the Headscale server (cryodev-main):
+
+```bash
+sudo headscale preauthkeys create --expiration 99y --reusable --user default
+```
+
+### Add to Secrets
+
+```bash
+sops hosts/<hostname>/secrets.yaml
+```
+
+```yaml
+tailscale:
+  auth-key: "your-preauth-key"
+```
+
+### Configuration
+
+```nix
+# In your host configuration
+{ config, ... }:
+{
+  sops.secrets."tailscale/auth-key" = { };
+  
+  services.tailscale = {
+    enable = true;
+    authKeyFile = config.sops.secrets."tailscale/auth-key".path;
+    extraUpFlags = [
+      "--login-server=https://headscale.cryodev.xyz"
+    ];
+  };
+}
+```
+
+## Usage
+
+### Check Status
+
+```bash
+tailscale status
+```
+
+### View IP Address
+
+```bash
+tailscale ip
+```
+
+### Ping Another Node
+
+```bash
+tailscale ping <hostname>
+```
+
+### SSH to Another Node
+
+```bash
+ssh user@<hostname>
+# or using Tailscale IP
+ssh user@100.64.0.X
+```
+
+## MagicDNS
+
+With Headscale's MagicDNS enabled, you can reach nodes by hostname:
+
+```bash
+ping cryodev-pi
+ssh steffen@cryodev-main
+```
+
+## Troubleshooting
+
+### Check Service Status
+
+```bash
+sudo systemctl status tailscaled
+```
+
+### View Logs
+
+```bash
+sudo journalctl -u tailscaled -f
+```
+
+### Re-authenticate
+
+If the node is not connecting:
+
+```bash
+sudo tailscale up --login-server=https://headscale.cryodev.xyz --force-reauth
+```
+
+### Node Not Appearing in Headscale
+
+Check the auth key is valid:
+
+```bash
+# On Headscale server
+sudo headscale preauthkeys list --user default
+```
+
+Verify the login server URL is correct in the client configuration.
--- a/flake.lock
+++ b/flake.lock
@ -0,0 +1,627 @@
+{
+  "nodes": {
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
+    "comin": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1772962094,
+        "narHash": "sha256-9+/PHrDNDUy9iiN7seOhcxq3KoVlCAmCim6HXuKTI24=",
+        "owner": "nlewo",
+        "repo": "comin",
+        "rev": "269ef4334f202b226eef804c0be0201891fb9c5d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nlewo",
+        "repo": "comin",
+        "type": "github"
+      }
+    },
+    "deploy-rs": {
+      "inputs": {
+        "flake-compat": "flake-compat_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "utils": "utils"
+      },
+      "locked": {
+        "lastModified": 1770019181,
+        "narHash": "sha256-hwsYgDnby50JNVpTRYlF3UR/Rrpt01OrxVuryF40CFY=",
+        "owner": "serokell",
+        "repo": "deploy-rs",
+        "rev": "77c906c0ba56aabdbc72041bf9111b565cdd6171",
+        "type": "github"
+      },
+      "original": {
+        "owner": "serokell",
+        "repo": "deploy-rs",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "nixpkgs": [
+          "headplane",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1768818222,
+        "narHash": "sha256-460jc0+CZfyaO8+w8JNtlClB2n4ui1RbHfPTLkpwhU8=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "255a2b1725a20d060f566e4755dbf571bbbb5f76",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1765121682,
+        "narHash": "sha256-4VBOP18BFeiPkyhy9o4ssBNQEvfvv1kXkasAYd0+rrA=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "65f23138d8d09a92e30f1e5c87611b23ef451bf3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1733328505,
+        "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-compat_4": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nixvim",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1768135262,
+        "narHash": "sha256-PVvu7OqHBGWN16zSi6tEmPwwHQ4rLPU9Plvs8/1TUBY=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "80daad04eddbbf5a4d883996a73f3f542fa437ac",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_3"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat_3",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772893680,
+        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "git-hooks_2": {
+      "inputs": {
+        "flake-compat": [
+          "nixos-mailserver",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore_2",
+        "nixpkgs": [
+          "nixos-mailserver",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1772893680,
+        "narHash": "sha256-JDqZMgxUTCq85ObSaFw0HhE+lvdOre1lx9iI6vYyOEs=",
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "8baab586afc9c9b57645a734c820e4ac0a604af9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_2": {
+      "inputs": {
+        "nixpkgs": [
+          "nixos-mailserver",
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "headplane": {
+      "inputs": {
+        "devshell": "devshell",
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1773108598,
+        "narHash": "sha256-y80AABZv5n1vQua8mn1T79QB4pRnBTo+hPdmPa+J0yA=",
+        "owner": "tale",
+        "repo": "headplane",
+        "rev": "6470f5a821e3ee5b4937a858bf13fb294bd38a7c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "tale",
+        "repo": "headplane",
+        "type": "github"
+      }
+    },
+    "ixx": {
+      "inputs": {
+        "flake-utils": [
+          "nixvim",
+          "nuschtosSearch",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixvim",
+          "nuschtosSearch",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1754860581,
+        "narHash": "sha256-EM0IE63OHxXCOpDHXaTyHIOk2cNvMCGPqLt/IdtVxgk=",
+        "owner": "NuschtOS",
+        "repo": "ixx",
+        "rev": "babfe85a876162c4acc9ab6fb4483df88fa1f281",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "ref": "v0.1.1",
+        "repo": "ixx",
+        "type": "github"
+      }
+    },
+    "nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat_4",
+        "git-hooks": "git-hooks_2",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1773194666,
+        "narHash": "sha256-YbsbqtTB3q0JjP7/G7GO58ea49cps1+8sb95/Bt7oVs=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "489fbc4e0ef987cfdce700476abafe3269ebf3e5",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1770107345,
+        "narHash": "sha256-tbS0Ebx2PiA1FRW8mt8oejR0qMXmziJmPaU1d4kYY9g=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "4533d9293756b63904b7238acb84ac8fe4c8c2c4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-old-stable": {
+      "locked": {
+        "lastModified": 1767313136,
+        "narHash": "sha256-16KkgfdYqjaeRGBaYsNrhPRRENs0qzkQVUooNHtoy2w=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ac62194c3917d5f474c1a844b6fd6da2db95077d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-25.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1772963539,
+        "narHash": "sha256-9jVDGZnvCckTGdYT53d/EfznygLskyLQXYwJLKMPsZs=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "9dcb002ca1690658be4a04645215baea8b95f31d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1772736753,
+        "narHash": "sha256-au/m3+EuBLoSzWUCb64a/MZq6QUtOV8oC0D9tY2scPQ=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "917fec990948658ef1ccd07cef2a1ef060786846",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1773068389,
+        "narHash": "sha256-vMrm7Pk2hjBRPnCSjhq1pH0bg350Z+pXhqZ9ICiqqCs=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "44bae273f9f82d480273bab26f5c50de3724f52f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-25.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixvim": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nuschtosSearch": "nuschtosSearch",
+        "systems": "systems_4"
+      },
+      "locked": {
+        "lastModified": 1769049374,
+        "narHash": "sha256-h0Os2qqNyycDY1FyZgtbn28VF1ySP74/n0f+LDd8j+w=",
+        "owner": "nix-community",
+        "repo": "nixvim",
+        "rev": "b8f76bf5751835647538ef8784e4e6ee8deb8f95",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "nixos-25.11",
+        "repo": "nixvim",
+        "type": "github"
+      }
+    },
+    "nuschtosSearch": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "ixx": "ixx",
+        "nixpkgs": [
+          "nixvim",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1768249818,
+        "narHash": "sha256-ANfn5OqIxq3HONPIXZ6zuI5sLzX1sS+2qcf/Pa0kQEc=",
+        "owner": "NuschtOS",
+        "repo": "search",
+        "rev": "b6f77b88e9009bfde28e2130e218e5123dc66796",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NuschtOS",
+        "repo": "search",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "comin": "comin",
+        "deploy-rs": "deploy-rs",
+        "git-hooks": "git-hooks",
+        "headplane": "headplane",
+        "nixos-mailserver": "nixos-mailserver",
+        "nixpkgs": "nixpkgs_3",
+        "nixpkgs-old-stable": "nixpkgs-old-stable",
+        "nixpkgs-unstable": "nixpkgs-unstable",
+        "nixvim": "nixvim",
+        "sops-nix": "sops-nix"
+      }
+    },
+    "sops-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1773096132,
+        "narHash": "sha256-M3zEnq9OElB7zqc+mjgPlByPm1O5t2fbUrH3t/Hm5Ag=",
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "rev": "d1ff3b1034d5bab5d7d8086a7803c5a5968cd784",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_4": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1770228511,
+        "narHash": "sha256-wQ6NJSuFqAEmIg2VMnLdCnUc0b7vslUohqqGGD+Fyxk=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "337a4fe074be1042a35086f15481d763b8ddc0e7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
+    },
+    "utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
--- a/flake.nix
+++ b/flake.nix
@ -10,7 +10,7 @@
    nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver";
    nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";

-    headplane.url = "github:yrd/headplane-nix";
+    headplane.url = "github:tale/headplane";

    comin.url = "github:nlewo/comin";
    comin.inputs.nixpkgs.follows = "nixpkgs";
@ -41,7 +41,9 @@

      forAllSystems = nixpkgs.lib.genAttrs supportedSystems;

-      lib = nixpkgs.lib;
+      # Extend nixpkgs.lib with our custom utils
+      lib = nixpkgs.lib.extend (final: prev: self.lib or { });
+
      constants = import ./constants.nix;

      mkNixosConfiguration =
@ -59,6 +61,26 @@
        };
    in
    {
+      # Custom library functions
+      lib = {
+        utils = import ./lib/utils.nix { lib = nixpkgs.lib; };
+      };
+
+      # Apps
+      apps = forAllSystems (
+        system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+          mkApp = name: {
+            type = "app";
+            program = pkgs.lib.getExe (pkgs.callPackage ./apps/${name} { });
+          };
+        in
+        {
+          rebuild = mkApp "rebuild";
+        }
+      );
+
      packages = forAllSystems (system: import ./pkgs nixpkgs.legacyPackages.${system});

      overlays = import ./overlays { inherit inputs; };
--- a/hosts/cryodev-main/binfmt.nix
+++ b/hosts/cryodev-main/binfmt.nix
@ -0,0 +1,4 @@
+# Enable QEMU emulation for aarch64 to build Raspberry Pi images
+{
+  boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
+}
--- a/hosts/cryodev-main/default.nix
+++ b/hosts/cryodev-main/default.nix
@ -1,11 +1,13 @@
 {
  inputs,
+  lib,
  outputs,
  ...
 }:

 {
  imports = [
+    ./binfmt.nix
    ./boot.nix
    ./hardware.nix
    ./networking.nix
@ -17,5 +19,12 @@
    outputs.nixosModules.nixvim
  ];

+  # Allow unfree packages (netdata has changed to gpl3Plus ncul1 license)
+  nixpkgs.config.allowUnfreePredicate =
+    pkg:
+    builtins.elem (lib.getName pkg) [
+      "netdata"
+    ];
+
  system.stateVersion = "25.11";
 }
--- a/hosts/cryodev-main/secrets.yaml
+++ b/hosts/cryodev-main/secrets.yaml
@ -0,0 +1,18 @@
+# SOPS encrypted secrets for cryodev-main
+# This file should be encrypted with sops before committing
+# See INSTRUCTIONS.md for setup instructions
+
+# Placeholder - replace with actual encrypted secrets
+forgejo-runner:
+    token: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+tailscale:
+    auth-key: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+headplane:
+    cookie_secret: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+    agent_pre_authkey: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+mailserver:
+    accounts:
+        forgejo: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+        admin: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+forgejo:
+    mail-pw: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
--- a/hosts/cryodev-main/services/forgejo.nix
+++ b/hosts/cryodev-main/services/forgejo.nix
@ -1,6 +1,5 @@
 {
  config,
-  pkgs,
  outputs,
  constants,
  ...
@ -31,14 +30,17 @@
        USER = "forgejo@${constants.domain}";
      };
    };
-    sops = true; # Enable sops integration for secrets
  };

  services.forgejo-runner = {
    enable = true;
    url = "https://${constants.services.forgejo.fqdn}";
-    # Token needs to be set up via sops/secrets
-    sops = true;
+    tokenFile = config.sops.secrets."forgejo-runner/token".path;
+  };
+
+  sops.secrets."forgejo-runner/token" = {
+    # gitea-runner user is created by gitea-actions-runner service
+    mode = "0400";
  };

  services.nginx.virtualHosts."${constants.services.forgejo.fqdn}" = {
--- a/hosts/cryodev-main/services/headplane.nix
+++ b/hosts/cryodev-main/services/headplane.nix
@ -1,6 +1,4 @@
 {
-  config,
-  pkgs,
  outputs,
  constants,
  ...
@ -14,14 +12,11 @@
  services.headplane = {
    enable = true;
    port = constants.services.headplane.port;
-    headscale = {
-      url = "http://127.0.0.1:${toString constants.services.headscale.port}";
-      public_url = "https://${constants.services.headscale.fqdn}";
-    };
-    # Secrets for headplane need to be configured via sops
-    sops.secrets = {
-      "headplane/cookie_secret" = { };
-      "headplane/agent_pre_authkey" = { };
+    settings = {
+      headscale = {
+        url = "http://127.0.0.1:${toString constants.services.headscale.port}";
+        public_url = "https://${constants.services.headscale.fqdn}";
+      };
    };
  };

--- a/hosts/cryodev-main/services/headscale.nix
+++ b/hosts/cryodev-main/services/headscale.nix
@ -1,6 +1,4 @@
 {
-  config,
-  pkgs,
  outputs,
  constants,
  ...
@ -17,7 +15,9 @@
    port = constants.services.headscale.port;
    settings = {
      server_url = "https://${constants.services.headscale.fqdn}";
-      dns_config.base_domain = constants.domain;
+      # dns.base_domain must be different from the server domain
+      # Using "tail" for internal Tailscale DNS (e.g., host.tail)
+      dns.base_domain = "tail";
    };
  };

--- a/hosts/cryodev-main/services/mailserver.nix
+++ b/hosts/cryodev-main/services/mailserver.nix
@ -1,6 +1,4 @@
 {
-  config,
-  pkgs,
  outputs,
  constants,
  ...
@ -21,7 +19,9 @@
        aliases = [ "postmaster" ];
      };
    };
-    certificateScheme = "acme-nginx";
-    sops = true;
+    x509.useACMEHost = constants.services.mail.fqdn;
  };
+
+  # ACME certificate for mail server
+  security.acme.certs.${constants.services.mail.fqdn} = { };
 }
--- a/hosts/cryodev-pi/default.nix
+++ b/hosts/cryodev-pi/default.nix
@ -1,5 +1,6 @@
 {
  inputs,
+  lib,
  outputs,
  ...
 }:
@ -10,12 +11,21 @@
    ./hardware.nix
    ./networking.nix
    ./packages.nix
+    ./sd-image.nix
    ./services
    ./users.nix

    outputs.nixosModules.common
    outputs.nixosModules.nixvim
+    outputs.nixosModules.sops
  ];

+  # Allow unfree packages (netdata has changed to gpl3Plus ncul1 license)
+  nixpkgs.config.allowUnfreePredicate =
+    pkg:
+    builtins.elem (lib.getName pkg) [
+      "netdata"
+    ];
+
  system.stateVersion = "25.11";
 }
--- a/hosts/cryodev-pi/sd-image.nix
+++ b/hosts/cryodev-pi/sd-image.nix
@ -0,0 +1,30 @@
+# SD Card image configuration for Raspberry Pi
+{
+  config,
+  modulesPath,
+  lib,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/installer/sd-card/sd-image-aarch64.nix")
+  ];
+
+  sdImage = {
+    # Compress with zstd for smaller download
+    compressImage = true;
+
+    # Auto-expand root partition on first boot
+    expandOnBoot = true;
+  };
+
+  # Image filename based on hostname
+  image.fileName = "${config.networking.hostName}-sd-image.img";
+
+  # Disable ZFS to avoid build issues on SD image
+  boot.supportedFilesystems = lib.mkForce [
+    "vfat"
+    "ext4"
+  ];
+}
--- a/hosts/cryodev-pi/secrets.yaml
+++ b/hosts/cryodev-pi/secrets.yaml
@ -0,0 +1,11 @@
+# SOPS encrypted secrets for cryodev-pi
+# This file should be encrypted with sops before committing
+# See INSTRUCTIONS.md for setup instructions
+
+# Placeholder - replace with actual encrypted secrets
+# Generate UUID with: uuidgen
+netdata:
+    stream:
+        child-uuid: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
+tailscale:
+    auth-key: ENC[AES256_GCM,data:placeholder,tag:placeholder,type:str]
--- a/hosts/cryodev-pi/services/netdata.nix
+++ b/hosts/cryodev-pi/services/netdata.nix
@ -1,7 +1,5 @@
 {
  config,
-  pkgs,
-  outputs,
  constants,
  ...
 }:
@ -9,23 +7,48 @@
 {
  services.netdata = {
    enable = true;
-    config = {
-      stream = {
-        enabled = "yes";
-        destination = "${constants.hosts.cryodev-main.ip}:${toString constants.services.netdata.port}";
-        "api key" = config.sops.placeholder."netdata/stream/child-uuid";
+    config.global = {
+      "debug log" = "syslog";
+      "access log" = "syslog";
+      "error log" = "syslog";
+    };
+    configDir = {
+      "stream.conf" = config.sops.templates."netdata/stream.conf".path;
+    };
+  };
+
+  sops =
+    let
+      owner = config.services.netdata.user;
+      group = config.services.netdata.group;
+      mode = "0400";
+      restartUnits = [ "netdata.service" ];
+    in
+    {
+      # generate with `uuidgen`
+      secrets."netdata/stream/child-uuid" = {
+        inherit
+          owner
+          group
+          mode
+          restartUnits
+          ;
+      };
+
+      templates."netdata/stream.conf" = {
+        inherit
+          owner
+          group
+          mode
+          restartUnits
+          ;
+        # child node
+        content = ''
+          [stream]
+          enabled = yes
+          destination = ${constants.hosts.cryodev-main.ip}:${builtins.toString constants.services.netdata.port}
+          api key = ${config.sops.placeholder."netdata/stream/child-uuid"}
+        '';
      };
    };
-  };
-
-  # Make sure sops is enabled/imported for this host to handle the secret
-  imports = [ outputs.nixosModules.sops ];
-
-  sops = {
-    defaultSopsFile = ../secrets.yaml;
-    secrets."netdata/stream/child-uuid" = {
-      owner = "netdata";
-      group = "netdata";
-    };
-  };
 }
--- a/lib/utils.nix
+++ b/lib/utils.nix
@ -0,0 +1,85 @@
+{ lib, ... }:
+
+let
+  inherit (lib)
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkOption
+    types
+    ;
+in
+{
+  isNotEmptyStr = str: builtins.isString str && str != "";
+
+  mkMailIntegrationOption = service: {
+    enable = mkEnableOption "Mail integration for ${service}.";
+    smtpHost = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = "SMTP host for sending emails.";
+    };
+  };
+
+  mkReverseProxyOption = service: subdomain: {
+    enable = mkEnableOption "Nginx reverse proxy for ${service}.";
+    subdomain = mkOption {
+      type = types.str;
+      default = subdomain;
+      description = "Subdomain for Nginx virtual host. Leave empty for root domain.";
+    };
+    forceSSL = mkOption {
+      type = types.bool;
+      default = true;
+      description = "Force SSL for Nginx virtual host.";
+    };
+  };
+
+  mkUrl =
+    {
+      fqdn,
+      ssl ? false,
+      port ? null,
+      path ? "",
+      ...
+    }:
+    let
+      protocol = if ssl then "https" else "http";
+      portPart = if port != null then ":${toString port}" else "";
+      pathPart = if path != "" then "/${path}" else "";
+    in
+    "${protocol}://${fqdn}${portPart}${pathPart}";
+
+  mkVirtualHost =
+    {
+      address ? "127.0.0.1",
+      port ? null,
+      socketPath ? null,
+      location ? "/",
+      ssl ? false,
+      proxyWebsockets ? true,
+      recommendedProxySettings ? true,
+      extraConfig ? "",
+      ...
+    }:
+    let
+      target =
+        if port != null then
+          "http://${address}:${builtins.toString port}"
+        else if socketPath != null then
+          "http://unix:${socketPath}"
+        else
+          null;
+    in
+    {
+      enableACME = ssl;
+      forceSSL = ssl;
+
+      locations = mkIf (target != null) {
+        "${location}" = {
+          proxyPass = mkDefault target;
+          inherit proxyWebsockets recommendedProxySettings extraConfig;
+        };
+      };
+    };
+}
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@ -3,6 +3,8 @@
  comin = import ./comin;
  forgejo = import ./forgejo;
  forgejo-runner = import ./forgejo-runner;
+  headplane = import ./headplane;
+  headscale = import ./headscale;
  mailserver = import ./mailserver;
  nixvim = import ./nixvim;
  normalUsers = import ./normalUsers;
--- a/modules/nixos/forgejo/default.nix
+++ b/modules/nixos/forgejo/default.nix
@ -18,31 +18,31 @@ in
 {
  config = mkIf cfg.enable {
    services.forgejo = {
-      database.type = "postgres";
-      lfs.enable = true;
+      database.type = mkDefault "postgres";
+      lfs.enable = mkDefault true;
      settings = {
        server = {
-          DOMAIN = "git.${config.networking.domain}";
-          PROTOCOL = "http";
-          ROOT_URL = "https://${settings.server.DOMAIN}/";
-          HTTP_ADDR = "0.0.0.0";
-          HTTP_PORT = 3456;
-          SSH_PORT = head config.services.openssh.ports;
+          DOMAIN = mkDefault "git.${config.networking.domain}";
+          PROTOCOL = mkDefault "http";
+          ROOT_URL = mkDefault "https://${settings.server.DOMAIN}/";
+          HTTP_ADDR = mkDefault "0.0.0.0";
+          HTTP_PORT = mkDefault 3456;
+          SSH_PORT = mkDefault (head config.services.openssh.ports);
        };
        service = {
-          DISABLE_REGISTRATION = true;
+          DISABLE_REGISTRATION = mkDefault true;
        };
        ui = {
-          DEFAULT_THEME = "forgejo-dark";
+          DEFAULT_THEME = mkDefault "forgejo-dark";
        };
        actions = {
-          ENABLED = true;
+          ENABLED = mkDefault true;
        };
        mailer = {
          ENABLED = mkDefault false;
-          SMTP_ADDR = "mail.${config.networking.domain}";
-          FROM = "git@${settings.server.DOMAIN}";
-          USER = "git@${settings.server.DOMAIN}";
+          SMTP_ADDR = mkDefault "mail.${config.networking.domain}";
+          FROM = mkDefault "git@${settings.server.DOMAIN}";
+          USER = mkDefault "git@${settings.server.DOMAIN}";
        };
      };
      secrets = {
--- a/modules/nixos/headplane/default.nix
+++ b/modules/nixos/headplane/default.nix
@ -2,31 +2,30 @@
  inputs,
  config,
  lib,
+  pkgs,
  ...
 }:

 let
  cfg = config.services.headplane;
-  domain = config.networking.domain;
-  subdomain = cfg.reverseProxy.subdomain;
-  fqdn = if (cfg.reverseProxy.enable && subdomain != "") then "${subdomain}.${domain}" else domain;
  headscale = config.services.headscale;

  inherit (lib)
    mkDefault
    mkIf
-    ;
-
-  inherit (lib.utils)
-    mkReverseProxyOption
-    mkVirtualHost
+    mkOption
+    types
    ;
 in
 {
  imports = [ inputs.headplane.nixosModules.headplane ];

  options.services.headplane = {
-    reverseProxy = mkReverseProxyOption "Headplane" "hp";
+    port = mkOption {
+      type = types.port;
+      default = 3000;
+      description = "Port for headplane to listen on";
+    };
  };

  config = mkIf cfg.enable {
@ -37,14 +36,14 @@ in
    services.headplane = {
      settings = {
        server = {
-          host = mkDefault (if cfg.reverseProxy.enable then "127.0.0.1" else "0.0.0.0");
-          port = mkDefault 3000;
+          host = mkDefault "127.0.0.1";
+          port = mkDefault cfg.port;
          cookie_secret_path = config.sops.secrets."headplane/cookie_secret".path;
        };
        headscale = {
-          url = "http://127.0.0.1:${toString headscale.port}";
-          public_url = headscale.settings.server_url;
-          config_path = "/etc/headscale/config.yaml";
+          url = mkDefault "http://127.0.0.1:${toString headscale.port}";
+          public_url = mkDefault headscale.settings.server_url;
+          config_path = mkDefault "/etc/headscale/config.yaml";
        };
        integration.agent = {
          enabled = mkDefault true;
@ -53,13 +52,6 @@ in
      };
    };

-    services.nginx.virtualHosts = mkIf cfg.reverseProxy.enable {
-      "${fqdn}" = mkVirtualHost {
-        port = cfg.settings.server.port;
-        ssl = cfg.reverseProxy.forceSSL;
-      };
-    };
-
    sops.secrets =
      let
        owner = headscale.user;
--- a/modules/nixos/headscale/default.nix
+++ b/modules/nixos/headscale/default.nix
@ -65,24 +65,26 @@ in
      address = mkDefault (if cfg.reverseProxy.enable then "127.0.0.1" else "0.0.0.0");
      port = mkDefault 8077;
      settings = {
-        policy.path = "/etc/${acl}";
-        database.type = "sqlite"; # postgres is highly discouraged as it is only supported for legacy reasons
-        server_url = mkUrl {
+        policy.path = mkDefault "/etc/${acl}";
+        database.type = mkDefault "sqlite"; # postgres is highly discouraged as it is only supported for legacy reasons
+        server_url = mkDefault (mkUrl {
          inherit fqdn;
          ssl = with cfg.reverseProxy; enable && forceSSL;
-        };
-        derp.server.enable = cfg.reverseProxy.forceSSL;
+        });
+        derp.server.enable = mkDefault cfg.reverseProxy.forceSSL;
        dns = {
          magic_dns = mkDefault true;
          base_domain = mkDefault "tail";
-          search_domains = [ cfg.settings.dns.base_domain ];
+          search_domains = mkDefault [ cfg.settings.dns.base_domain ];
          override_local_dns = mkDefault true;
-          nameservers.global = optionals cfg.settings.dns.override_local_dns [
-            "1.1.1.1"
-            "1.0.0.1"
-            "2606:4700:4700::1111"
-            "2606:4700:4700::1001"
-          ];
+          nameservers.global = mkDefault (
+            optionals cfg.settings.dns.override_local_dns [
+              "1.1.1.1"
+              "1.0.0.1"
+              "2606:4700:4700::1111"
+              "2606:4700:4700::1001"
+            ]
+          );
        };
      };
    };
--- a/modules/nixos/mailserver/default.nix
+++ b/modules/nixos/mailserver/default.nix
@ -66,8 +66,10 @@ in
      fqdn = mkDefault fqdn;

      domains = mkDefault [ domain ];
-      certificateScheme = mkDefault "acme-nginx";
-      stateVersion = mkDefault 1;
+      # stateVersion 3 requires the new mail directory structure
+      # For new installations, this is the correct value
+      # For existing installations, see: https://nixos-mailserver.readthedocs.io/en/latest/migrations.html
+      stateVersion = mkDefault 3;

      loginAccounts = mapAttrs' (
        user: accConf:
@ -79,8 +81,14 @@ in
          hashedPasswordFile = config.sops.secrets."mailserver/accounts/${user}".path;
        }
      ) cfg.accounts;
+
+      # Use ACME for certificate
+      x509.useACMEHost = mkDefault fqdn;
    };

+    # ACME certificate for mail server
+    security.acme.certs.${fqdn} = { };
+
    security.acme = {
      acceptTerms = true;
      defaults.email = mkDefault "postmaster@cryodev.xyz";
--- a/modules/nixos/nixvim/default.nix
+++ b/modules/nixos/nixvim/default.nix
@ -15,7 +15,9 @@ in
    inputs.nixvim.nixosModules.nixvim
    ./plugins

-    ./spellfiles.nix
+    # TODO: spellfiles.nix uses home-manager options (home.file, xdg.dataHome)
+    # which are not available in NixOS modules. Needs to be rewritten.
+    # ./spellfiles.nix
  ];

  config = {
--- a/modules/nixos/nixvim/plugins/lsp.nix
+++ b/modules/nixos/nixvim/plugins/lsp.nix
@ -64,6 +64,6 @@ in
      };
    };

-    home.packages = optional (cfg.enable && plugin.servers.nixd.enable) pkgs.nixfmt;
+    environment.systemPackages = optional (cfg.enable && plugin.servers.nixd.enable) pkgs.nixfmt;
  };
 }
--- a/modules/nixos/nixvim/plugins/telescope.nix
+++ b/modules/nixos/nixvim/plugins/telescope.nix
@ -45,7 +45,7 @@ in
      ];
    };

-    home.packages = optionals plugin.enable [
+    environment.systemPackages = optionals plugin.enable [
      pkgs.ripgrep # for "live_grep"
    ];
  };
--- a/modules/nixos/nixvim/plugins/treesitter.nix
+++ b/modules/nixos/nixvim/plugins/treesitter.nix
@ -30,12 +30,12 @@ in
    };

    # Fix for: ERROR `cc` executable not found.
-    home.sessionVariables = mkIf plugin.enable {
+    environment.sessionVariables = mkIf plugin.enable {
      CC = mkDefault cc;
    };

    # Fix for: WARNING `tree-sitter` executable not found
-    home.packages = mkIf plugin.enable [
+    environment.systemPackages = mkIf plugin.enable [
      plugin.package
    ];
  };
--- a/modules/nixos/sops/default.nix
+++ b/modules/nixos/sops/default.nix
@ -7,7 +7,16 @@
 }:

 let
-  secrets = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets/secrets.yaml";
+  # Check both locations for secrets.yaml
+  secretsInSubdir = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets/secrets.yaml";
+  secretsInRoot = "${toString inputs.self}/hosts/${config.networking.hostName}/secrets.yaml";
+  secrets =
+    if builtins.pathExists secretsInSubdir then
+      secretsInSubdir
+    else if builtins.pathExists secretsInRoot then
+      secretsInRoot
+    else
+      null;
 in
 {
  imports = [ inputs.sops-nix.nixosModules.sops ];
@ -17,5 +26,5 @@ in
    sops
  ];

-  sops.defaultSopsFile = lib.mkIf (builtins.pathExists secrets) (lib.mkDefault secrets);
+  sops.defaultSopsFile = lib.mkIf (secrets != null) (lib.mkDefault secrets);
 }