devachine.sh

#!/usr/bin/env bash
set -euo pipefail

DOTFILES_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# --- Secrets ---
export HCLOUD_TOKEN="${HCLOUD_TOKEN:-$(op read "op://Hetzner/token/credential")}"

# --- Configuration (override via env) ---
SERVER_NAME="${SERVER_NAME:-devachine}"

SSH_OPTS=(-o StrictHostKeyChecking=no -o ConnectTimeout=5)

# --- Subcommands ---
case "${1:-}" in
  down)
    echo ":: Deleting $SERVER_NAME (volume preserved)"
    hcloud server delete "$SERVER_NAME"
    exit 0
    ;;
  ssh)
    SERVER_IP=$(hcloud server ip "$SERVER_NAME")
    # Args: optional [user] and/or [--no-forward] in any order.
    # user@devachine overrides the alias's User directive while keeping
    # everything else (forwards, agent fwd) from ~/.ssh/config.
    SSH_USER="dian"
    NO_FORWARD=0
    for arg in "${@:2}"; do
      case "$arg" in
        --no-forward) NO_FORWARD=1 ;;
        --*) echo "error: unknown flag '$arg'"; exit 1 ;;
        *) SSH_USER="$arg" ;;
      esac
    done
    if (( NO_FORWARD )); then
      exec ssh "${SSH_OPTS[@]}" -o ClearAllForwardings=yes \
        -o Hostname="$SERVER_IP" "$SSH_USER@devachine"
    fi
    exec autossh -M 0 -o ServerAliveInterval=30 -o ServerAliveCountMax=3 \
      -o Hostname="$SERVER_IP" "$SSH_USER@devachine" -t "tmux new -A -s main"
    ;;
  scp)
    shift
    SERVER_IP=$(hcloud server ip "$SERVER_NAME")
    # Override Hostname for the 'devachine' ssh alias so users can write
    # `devachine.sh scp file devachine:~/` with the current dynamic IP.
    exec scp "${SSH_OPTS[@]}" -o Hostname="$SERVER_IP" "$@"
    ;;
  status)
    echo ":: Server"
    hcloud server list -l devachine -o columns=name,status,ipv4,server_type,datacenter 2>/dev/null \
      || echo "  (none)"
    echo ""
    echo ":: Volumes"
    hcloud volume list -l devachine -o columns=name,size,server,location 2>/dev/null \
      || echo "  (none)"
    echo ""
    echo ":: Snapshots"
    hcloud image list --type snapshot -l devachine -o columns=id,description,created,image_size 2>/dev/null \
      || echo "  (none)"
    exit 0
    ;;
  deploy)
    SERVER_IP=$(hcloud server ip "$SERVER_NAME")
    echo ":: Syncing dotfiles to $SERVER_IP"
    echo ":: Syncing dotfiles via tar..."
    tar -C "$DOTFILES_DIR" --exclude='.git' --exclude='result' --no-xattrs --no-mac-metadata --no-fflags -cf - . | \
      ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" "mkdir -p ~/dotfiles && tar -C ~/dotfiles -xf -"
    echo ":: Sync complete"

    echo ":: Applying NixOS configuration"
    # The heredoc just kicks off nixos-rebuild in the background and
    # exits; streaming + exit-code check happens below.
    ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" bash <<'DEPLOY'
      set -euo pipefail
      command -v git >/dev/null || nix-env -f '<nixpkgs>' -iA git
      cp /etc/nixos/hardware-configuration.nix ~/dotfiles/hosts/devachine-hardware.nix

      # Volume attachment can shift /dev/sdX names — point GRUB at the actual boot disk
      BOOT_DISK="/dev/$(lsblk -ndo PKNAME "$(findmnt -no SOURCE /)")"
      sed -i "s|boot.loader.grub.device = \"[^\"]*\"|boot.loader.grub.device = \"$BOOT_DISK\"|" \
        ~/dotfiles/hosts/devachine-hardware.nix

      # Some codepaths (manual edits, stale configs) drop fsType from the
      # root filesystem, which then fails evaluation deep in a trace. Heal
      # the generated hardware config in place using the live fs type.
      ROOT_FSTYPE=$(findmnt -no FSTYPE /)
      if ! grep -q 'fileSystems\."/"[^}]*fsType' ~/dotfiles/hosts/devachine-hardware.nix; then
        sed -i "s|fileSystems\.\"/\" = { device = \"\([^\"]*\)\"; };|fileSystems.\"/\" = { device = \"\1\"; fsType = \"$ROOT_FSTYPE\"; };|" \
          ~/dotfiles/hosts/devachine-hardware.nix
      fi

      VOLUME_DEV=$(ls /dev/disk/by-id/scsi-0HC_Volume_* 2>/dev/null | head -1)
      if [[ -n "$VOLUME_DEV" ]]; then
        blkid "$VOLUME_DEV" &>/dev/null || mkfs.ext4 -L devachine-data "$VOLUME_DEV"
        cat > ~/dotfiles/hosts/devachine-disks.nix <<EOF
{ ... }: {
  fileSystems."/home" = {
    device = "$VOLUME_DEV";
    fsType = "ext4";
    options = [ "defaults" "nofail" ];
  };
}
EOF
      fi

      cd ~/dotfiles
      git config --global --add safe.directory ~/dotfiles
      git init -q
      git add -A
      rm -f /root/rebuild.rc /root/rebuild.pid
      # Run nixos-rebuild in the background so the session can drop
      # (e.g. sshd restart). Exit code goes to rebuild.rc as a sentinel:
      # its presence means "finished", its value means "success/fail".
      nohup bash -c 'nixos-rebuild switch --flake .#devachine; echo $? > /root/rebuild.rc' \
        > /root/rebuild.log 2>&1 &
      echo $! > /root/rebuild.pid
      disown
DEPLOY

    # Stream the rebuild log; `tail --pid` exits cleanly when the
    # nohup'd rebuild process dies. Reconnect on SSH drop.
    echo ":: Streaming rebuild log"
    until ssh "${SSH_OPTS[@]}" -o ServerAliveInterval=10 -o ServerAliveCountMax=3 \
        root@"$SERVER_IP" 'tail -f /root/rebuild.log --pid=$(cat /root/rebuild.pid)'; do
      echo ":: SSH dropped, reconnecting..."
      sleep 3
    done

    # Small race between process exit and .rc write — give it a moment
    RC=""
    for _ in $(seq 1 10); do
      RC=$(ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" 'cat /root/rebuild.rc 2>/dev/null' || true)
      [[ -n "$RC" ]] && break
      sleep 1
    done
    if [[ "$RC" != "0" ]]; then
      echo "error: nixos-rebuild failed (exit ${RC:-unknown})"
      exit 1
    fi

    # Set up user home dirs + shared dotfiles dir on the volume.
    # /home/dotfiles is group-writable (setgid) so all users can edit
    # settings via Claude Code's /config, which writes through
    # ~/.claude/settings.json → ~/dotfiles → /home/dotfiles.
    ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" bash <<'POSTDEPLOY'
      set -euo pipefail
      for user in dian work; do
        mkdir -p "/home/$user"
        chown -R "$user:users" "/home/$user"
      done
      install -d -o dian -g users -m 2775 /home/dotfiles
      # Reset shared workdir to dian:users with group-writable perms.
      # work-user edits via Claude Code create files owned by them with
      # umask-default mode (no group write), which then breaks dian's
      # next `git pull`. Re-normalize on every deploy.
      if [[ -d /home/dotfiles && "$(ls -A /home/dotfiles)" ]]; then
        chown -R dian:users /home/dotfiles
        chmod -R u+rwX,g+rwX /home/dotfiles
        find /home/dotfiles -type d -exec chmod g+s {} +
      fi
POSTDEPLOY

    echo ":: nixos-rebuild complete"

    echo ":: Waiting for SSH as dian"
    for _ in $(seq 1 12); do
      ssh "${SSH_OPTS[@]}" -o BatchMode=yes dian@"$SERVER_IP" true 2>/dev/null && break
      sleep 5
    done

    echo ":: Cloning dotfiles to /home/dotfiles"
    # safe.directory whitelist for /home/dotfiles is declared in
    # home/common.nix — HM writes it into ~/.config/git/config (a
    # nix-store symlink, so `git config --global` doesn't work here).
    ssh -A "${SSH_OPTS[@]}" dian@"$SERVER_IP" bash <<'CLONE'
      set -euo pipefail
      ssh-keyscan -t ed25519 github.com >> ~/.ssh/known_hosts 2>/dev/null
      # /home/dotfiles is shared (setgid, group-writable) so files inside
      # may be owned by the work user after they edit via Claude Code, and
      # an older clone may have been done as root. Either trips git's
      # owner check. Whitelist the path so dian can run git here.
      git config --global --add safe.directory /home/dotfiles
      if [[ ! -d /home/dotfiles/.git ]]; then
        git clone git@github.com:dianfishekqi/dotfiles.git /home/dotfiles
      else
        cd /home/dotfiles && git pull
      fi
      sudo cp /root/dotfiles/hosts/devachine-hardware.nix /home/dotfiles/hosts/
      sudo cp /root/dotfiles/hosts/devachine-disks.nix /home/dotfiles/hosts/
CLONE

    echo ":: Linking ~/dotfiles → /home/dotfiles for each user"
    ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" bash <<'LINK'
      set -euo pipefail
      for user in dian work; do
        link="/home/$user/dotfiles"
        # Skip if user already has a real dotfiles dir (don't clobber work)
        if [[ -d "$link" && ! -L "$link" ]]; then
          echo "  $user: real dotfiles dir exists, skipping"
          continue
        fi
        ln -sfn /home/dotfiles "$link"
        chown -h "$user:users" "$link"
      done
LINK

    echo ":: Done! ssh devachine"
    exit 0
    ;;
  bootstrap) ;; # fall through to full bootstrap below
  *)
    echo "Usage: devachine.sh <command>"
    echo ""
    echo "Commands:"
    echo "  bootstrap   Create server, nix-infect, deploy flake config"
    echo "  deploy      Sync dotfiles + nixos-rebuild (no recreate)"
    echo "  down        Delete server (volume preserved)"
    echo "  ssh [--no-forward]  Connect with agent forwarding (optionally without port forwards)"
    echo "  scp ARGS    Run scp with the current server IP (use devachine: host)"
    echo "  status      Show server state + IP"
    exit 0
    ;;
esac
SERVER_TYPE="${SERVER_TYPE:-cpx42}"
LOCATION="${LOCATION:-fsn1}"
SSH_KEY="${SSH_KEY:-Dian Generic}"
VOLUME_SIZE="${VOLUME_SIZE:-50}"
VOLUME_NAME="${VOLUME_NAME:-devachine-data}"
SNAPSHOT_DESC="devachine-base-nixos"

# --- Preflight ---
command -v hcloud >/dev/null 2>&1 || { echo "error: hcloud CLI not found"; exit 1; }

echo ":: Using SSH key '$SSH_KEY'"

# --- Volume (reuse or create) ---
if hcloud volume describe "$VOLUME_NAME" &>/dev/null; then
  echo ":: Volume '$VOLUME_NAME' exists, reusing"
else
  echo ":: Creating ${VOLUME_SIZE}GB volume in $LOCATION"
  hcloud volume create --name "$VOLUME_NAME" --size "$VOLUME_SIZE" \
    --location "$LOCATION" --format ext4 --label devachine=
fi

# --- Pick image: reuse NixOS snapshot if available, otherwise Ubuntu + nix-infect ---
SNAPSHOT_ID=$(hcloud image list --type snapshot -l devachine -o noheader -o columns=id 2>/dev/null | head -1)

wait_ssh() {
  echo ":: Waiting for SSH on $SERVER_IP"
  for _ in $(seq 1 "$1"); do
    ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" true 2>/dev/null && return 0
    sleep 3
  done
  echo "error: SSH timeout"; exit 1
}

if [[ -n "$SNAPSHOT_ID" ]]; then
  echo ":: Using existing NixOS snapshot ($SNAPSHOT_ID)"
  IMAGE="$SNAPSHOT_ID"
else
  echo ":: No NixOS snapshot found, will nix-infect from Ubuntu"
  IMAGE="ubuntu-22.04"
fi

# --- Server ---
echo ":: Creating $SERVER_TYPE server '$SERVER_NAME' in $LOCATION"
hcloud server create \
  --name "$SERVER_NAME" \
  --type "$SERVER_TYPE" \
  --image "$IMAGE" \
  --location "$LOCATION" \
  --ssh-key "$SSH_KEY" \
  --volume "$VOLUME_NAME" \
  --label devachine=

SERVER_IP=$(hcloud server ip "$SERVER_NAME")
echo ":: Server IP: $SERVER_IP"

echo ":: Clearing old host key for $SERVER_IP"
ssh-keygen -R "$SERVER_IP" 2>/dev/null || true
echo ":: Waiting for SSH..."
wait_ssh 30
echo ":: SSH ready"

# --- nix-infect (only if created from Ubuntu) ---
if [[ -z "$SNAPSHOT_ID" ]]; then
  echo ":: Running nix-infect (server will reboot automatically)"
  # nix-infect calls reboot at the end — SSH connection drops.
  # || true prevents set -e from killing the script; we verify NixOS below.
  rc=0
  ssh "${SSH_OPTS[@]}" -o ServerAliveInterval=5 -o ServerAliveCountMax=3 \
    root@"$SERVER_IP" bash <<'REMOTE' || rc=$?
    set -euo pipefail
    export DEBIAN_FRONTEND=noninteractive
    apt-get update -qq && apt-get install -y -qq curl > /dev/null
    curl -fsSL https://github.com/elitak/nixos-infect/raw/master/nixos-infect | \
      NIX_CHANNEL=nixos-25.05 bash -x 2>&1 | tail -20
REMOTE

  echo ":: Server rebooting (ssh exited $rc)"
  ssh-keygen -R "$SERVER_IP" 2>/dev/null || true
  sleep 10
  wait_ssh 60

  OS_ID=$(ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" "grep '^ID=' /etc/os-release" 2>/dev/null)
  if [[ "$OS_ID" != "ID=nixos" ]]; then
    echo "error: nix-infect failed — server is still running ${OS_ID#ID=}"
    exit 1
  fi
  echo ":: NixOS confirmed"

  # Label root filesystem so the snapshot is portable across servers
  # (device names like /dev/sda1 change, labels don't)
  echo ":: Labeling root filesystem for portable snapshots"
  ssh "${SSH_OPTS[@]}" root@"$SERVER_IP" bash <<'LABEL'
    set -euo pipefail
    ROOT_DEV=$(findmnt -no SOURCE /)
    e2label "$ROOT_DEV" nixos
    # Only replace the fileSystems root device, not the GRUB boot.loader device
    sed -i '/fileSystems\."\/"/,/};/ s|device = "[^"]*";|device = "/dev/disk/by-label/nixos";|' \
      /etc/nixos/hardware-configuration.nix
    # Volume attachment can shift /dev/sdX names — point GRUB at the actual boot disk
    BOOT_DISK="/dev/$(lsblk -ndo PKNAME "$ROOT_DEV")"
    sed -i "s|boot.loader.grub.device = \"[^\"]*\"|boot.loader.grub.device = \"$BOOT_DISK\"|" \
      /etc/nixos/hardware-configuration.nix
    nixos-rebuild switch
LABEL

  echo ":: Creating base NixOS snapshot"
  hcloud server create-image --type snapshot --description "$SNAPSHOT_DESC" \
    --label devachine= "$SERVER_NAME"
fi

# --- Deploy flake config ---
exec "$0" deploy