infect:config:sha1: update nixos-config

Now you need to edit nixos-infect to update base configuration URL.

.drone.yml

@@ -16,18 +16,46 @@ steps:
       from_secret: CLOUDFLARE_TOKEN
     USER_PASS:
       from_secret: USER_PASS
-    INFECT_SOURCE_BRANCH: ${DRONE_SOURCE_BRANCH}
+    INFECT_COMMIT_SHA: ${DRONE_COMMIT_SHA}
 
   commands:
+  - set -o nounset
+  - > # TODO pass Base64 encoded password from Drone instead of this
+    ENCODED_PASSWORD="$(base64 <<<"$USER_PASS")"
   # Create infect user script and then push it to a remote machine on server creation.
   - |
     cat << EOF > infect.sh
     #! /usr/bin/env bash
 
-    export STAGING_ACME=true
+    set -o allexport
+    set -o errtrace
+    set -o pipefail
+    shopt -s inherit_errexit
+    trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR
+
+    API_TOKEN="$USER_PASS"
+    DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN
+    DNS_PROVIDER_TYPE=CLOUDFLARE
+    DOMAIN=$DOMAIN
+    ENCODED_PASSWORD="$ENCODED_PASSWORD"
+    HOSTNAME=selfprivacy-ci-test
+    LUSER=cicdcicd
+    NIXOS_CONFIG_ID=default
+    PROVIDER=HETZNER
+    SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K"
+    STAGING_ACME=true
+
+    curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \
+    | bash 2>&1 | tee /root/infect.log
     EOF
-  - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh
+  - cat infect.sh
-  - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1
+  - location=fsn1
+  - set -o pipefail
+  - >
+    volume_id="$(http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" format=ext4 size:=10 location=$location name=10gb-by-drone-ci automount:=false | jq -re .volume.id)"
+  # TODO pass linux device volume_id name to infect.sh script
+  - >
+    http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 volumes:=[$volume_id] user_data=@infect.sh automount:=false location=$location
 
 - name: dns
   environment:
@@ -110,11 +138,11 @@ steps:
   - curl https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -sS --fail -o root.pem
   - dig api.$DOMAIN
   - >
-    retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS"
+    retry $TIMEOUT_1 10 curl https://api.$DOMAIN/api/version -sS --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS"
   - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp
   - >
     curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"'
-  - echo 'File to test nextcloud upload' > file
+  - echo 'File contents to test nextcloud upload' > file
   - >
     curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -d @file -u "admin:$USER_PASS"
   - >
@@ -127,7 +155,7 @@ steps:
     API_TOKEN:
       from_secret: USER_PASS
     TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git
-    TIMEOUT_1: 100
+    TIMEOUT_1: 120
   commands:
   - source $(nix build $TESTS_REPO#library --print-out-paths)
   - retry $TIMEOUT_1 10 query-minimum-services
@@ -137,7 +165,7 @@ steps:
   - enable-service ocserv
   - enable-service pleroma
   - nixos-rebuild Rebuild
-  - sleep 100
+  - sleep 180
   - retry $TIMEOUT_1 10 query-all-services
   - nixos-rebuild Rollback
   - sleep 30
@@ -155,7 +183,7 @@ steps:
   environment:
     DOMAIN:
       from_secret: DOMAIN
-    TIMEOUT: 3000
+    TIMEOUT: 1500
   commands:
     - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done
   when:
@@ -190,6 +218,16 @@ steps:
     echo "$ZONE_ID"
   - >
     echo "$result" | jq '.[] | "\(.id)"' | while read id ; do echo "Deleting $id"; http -v --check-status --ignore-stdin DELETE https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$(echo $id | cut -f 2 -d '"') Authorization:"Bearer $CLOUDFLARE_TOKEN"; done
+  - location="fsn1"
+  - volume_name="10gb-by-drone-ci"
+  - >
+    volumes_json="$(http -d --print=HBb --ignore-stdin GET https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" name=$volume_name)"
+    # as of 2023-12-27 Hetzner ignores "name" query parameter (and others too)!!
+  - >
+    volume_id="$(jq <<<"$volumes_json" -re "(.volumes[] | select(.name == \"$volume_name\")).id")"
+  - sleep 1
+  - >
+    http -d --print=HBb --ignore-stdin DELETE https://api.hetzner.cloud/v1/volumes/$volume_id Authorization:"Bearer $PASSWORD"
   when:
     status:
     - failure

nixos-infect

@@ -4,20 +4,24 @@
 # - https://github.com/elitak/nixos-infect
 # - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect
 
+# mandatory variables
 : "${API_TOKEN:?API_TOKEN variable is not set}"
-: "${DOMAIN:?DOMAIN variable is not set}"
-: "${LUSER:?LUSER variable is not set}"
-: "${HOSTNAME:?HOSTNAME variable is not set}"
-: "${PROVIDER:?PROVIDER variable is not set}"
-: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}"
-: "${STAGING_ACME:?STAGING_ACME variable is not set}"
 : "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}"
-: "${DB_PASSWORD:?DB_PASSWORD variable is not set}"
+: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}"
-: "${USER_PASS:?USER_PASS variable is not set}"
+: "${DOMAIN:?DOMAIN variable is not set}"
-: "${NIX_VERSION:?NIX_VERSION variable is not set}"
+: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}"
-: "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}"
+: "${HOSTNAME:?HOSTNAME variable is not set}"
-: "${CONFIG_URL:?CONFIG_URL variable is not set}"
+: "${LUSER:?LUSER variable is not set}"
+: "${PROVIDER:?PROVIDER variable is not set}"
+
+# optional variables
+: "${NIXOS_CONFIG_ID:=default}"
 : "${SSH_AUTHORIZED_KEY:=}"
+: "${VOLUME_DEV_PATH:=}"
+: "${STAGING_ACME:=false}"
+
+readonly CONFIG_URL="https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/c5b16ac16dd328199e8f4f901890fff63f2fb81f.tar.gz"
+readonly NIX_VERSION="2.18.1"
 
 readonly NL=$'\n'
 readonly LOCAL_FLAKE_DIR="/etc/nixos"
@@ -27,15 +31,41 @@ readonly NIX_OPTS=(--extra-experimental-features "nix-command flakes")
 DoNetConf=
 
 genOptionalSsh() {
-  [ -n "${SSH_AUTHORIZED_KEY}" ] && cat << EOF
+  if [ -n "${SSH_AUTHORIZED_KEY}" ]; then
+      cat << EOF
 "ssh": { "rootKeys": [ "${SSH_AUTHORIZED_KEY}" ] },
 EOF
+  fi
+}
+
+# TODO receive disk device path from outside
+determine2ndDisk() {
+  local volDevPath rootFsDev
+  if [ -n "$VOLUME_DEV_PATH" ]; then
+      volDevPath="$VOLUME_DEV_PATH"
+  else
+      case "$PROVIDER" in
+          HETZNER)
+              volDevPath="/dev/sdb"
+              ;;
+          DIGITALOCEAN)
+              volDevPath="/dev/sda"
+              ;;
+          *)
+              return 1
+              ;;
+      esac
+  fi
+  rootFsDev="$(findmnt -no source -T /)"
+  [[ -b "$volDevPath" && "$volDevPath" != "$rootFsDev" ]] && echo "$volDevPath"
 }
 
 # Merge original userdata.json with deployment specific fields and print result.
 genUserdata() {
-  local HASHED_PASSWORD userdata_infect release
+  local hashed_password diskDeviceName userdata_infect
-  HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")"
+  hashed_password="$(mkpasswd -m sha-512 "$USER_PASS")"
+  diskDevice="$(determine2ndDisk)"
+  diskDeviceName="${diskDevice##/dev/}"
 
   userdata_infect=$(cat << EOF
 {
@@ -48,9 +78,33 @@ genUserdata() {
         "provider": "$PROVIDER"
     },
     "domain": "$DOMAIN",
-    "hashedMasterPassword": "$HASHED_PASSWORD",
+    "hashedMasterPassword": "$hashed_password",
     "hostname": "$HOSTNAME",
-    "username": "$LUSER"
+    "username": "$LUSER",
+    "volumes": [
+        {
+            "device": "$diskDevice",
+            "mountPoint": "/volumes/$diskDeviceName",
+            "fsType": "ext4"
+        }
+    ],
+    "modules": {
+        "bitwarden": {
+            "location": "$diskDeviceName"
+        },
+        "gitea": {
+            "location": "$diskDeviceName"
+        },
+        "nextcloud": {
+            "location": "$diskDeviceName"
+        },
+        "pleroma": {
+            "location": "$diskDeviceName"
+        },
+        "simple-nixos-mailserver": {
+            "location": "$diskDeviceName"
+        }
+    }
 }
 EOF
 )
@@ -60,13 +114,16 @@ EOF
 }
 
 genSecrets() {
+  local dbpass
+  dbpass="$(shuf --random-source=/dev/urandom -erz -n32 {A..Z} {a..z} {0..9} | tr -d '\n')"
+
   cat << EOF
 {
     "api": {
         "token": "$API_TOKEN",
         "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"]
     },
-    "databasePassword": "$DB_PASSWORD",
+    "databasePassword": "$dbpass",
     "dns": {
         "apiKey": "$DNS_PROVIDER_TOKEN"
     },
@@ -109,7 +166,7 @@ EOF
   # FIXME it's questionable whether these modules are needed at all...
   declare -a availableKernelModules=()
 
-  [ "$PROVIDER" == "digitalocean" ] \
+  [ "$PROVIDER" == "DIGITALOCEAN" ] \
   && availableKernelModules+=('"ata_piix"' '"uhci_hcd"' '"xen_blkfront"')
 
   [ "$(uname -m)" == "x86_64" ] \
@@ -153,7 +210,7 @@ EOF
 
 setupConf() {
   mkdir -p ${LOCAL_FLAKE_DIR}
-  if ! curl "${CONFIG_URL}" \
+  if ! curl --fail "${CONFIG_URL}" \
   | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 --exclude=".*"
   then
       echo "Error downloading/extracting top level flake configuration!"
@@ -293,6 +350,12 @@ findESP() {
 }
 
 prepareEnv() {
+  if ! USER_PASS="$(base64 -d <<<"$ENCODED_PASSWORD")"; then
+      echo "Error decoding ENCODED_PASSWORD from Base64!"
+      exit 1
+  fi
+  readonly USER_PASS
+
   isEFI=0
   [ -d /sys/firmware/efi ] && isEFI=1
 
@@ -368,7 +431,8 @@ checkEnv() {
   req xzcat            || { echo "ERROR: Missing xzcat";               return 1; }
   req awk              || { echo "ERROR: Missing awk";                 return 1; }
   req cut || req df    || { echo "ERROR: Missing coreutils (cut, df)"; return 1; }
-  req mkpasswd         || { echo "ERROR: Missing mkpasswd"; return 1; }
+  req mkpasswd         || { echo "ERROR: Missing mkpasswd";            return 1; }
+  req shuf             || { echo "ERROR: Missing shuf";                return 1; }
 }
 
 # Download and execute the nix installer script.
@@ -389,12 +453,12 @@ installNix() {
   fi
 
   echo "Downloading install script from ${installURL}..."
-  if ! curl "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then
+  if ! curl --fail "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then
       echo "Failure while downloading Nix install script!"
       return 1
   fi
 
-  if ! sha="$(curl "${shaURL}")"; then
+  if ! sha="$(curl --fail "${shaURL}")"; then
       echo "Failure while downloading Nix install script sha!"
       return 1
   fi
@@ -417,17 +481,16 @@ installNix() {
 }
 
 infect() {
-  # this is needed solely for accepting the sp-module subflake
+  # this is needed solely for accepting the sp-module sub-flake
   # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478
-  /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \
+  cd /etc/nixos || return 1
-  --extra-experimental-features "nix-command flakes" \
+  $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} \
-  --override-input sp-modules ${LOCAL_FLAKE_DIR}/sp-modules
+  --override-input sp-modules path:./sp-modules
 
   echo "nix build the configuration flake..."
-  if ! /root/.nix-profile/bin/nix build \
+  if ! $NIX "${NIX_OPTS[@]}" build \
-  --extra-experimental-features "nix-command flakes" \
   --profile /nix/var/nix/profiles/system \
-  ${LOCAL_FLAKE_DIR}/#nixosConfigurations.sp-nixos.config.system.build.toplevel
+  ${LOCAL_FLAKE_DIR}/#nixosConfigurations."$NIXOS_CONFIG_ID".config.system.build.toplevel
   then
       echo "Failed!"
       exit 1
@@ -457,20 +520,20 @@ infect() {
   fi
 
   # Remove nix installed by the "install" script.
-  rm -fv /nix/var/nix/profiles/default*
+  rm -fv /nix/var/nix/profiles/default* /root/.nix-profile
+  rm -rv /nix/var/nix/profiles/per-user
   /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage
 }
 
-set -o pipefail
+set -o errtrace
 set -o nounset
-set -o errexit
+set -o pipefail
 set -o xtrace
 shopt -s inherit_errexit
-
+trap 'echo ${LINENO}: "$BASH_COMMAND"; exit 1' ERR
-genNetworkingConf
 
 # digitalocean requires detailed network config to be generated
-[ "$PROVIDER" == "digitalocean" ] && DoNetConf="y"
+[ "$PROVIDER" == "DIGITALOCEAN" ] && DoNetConf="y"
 
 apt update
 apt install -y git tar curl whois jq