jsonnet test

2023-12-25 19:37:26 +04:00
2 changed files with 48 additions and 330 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -1,234 +1,15 @@
-kind: pipeline
-type: exec
-name: default
-
-clone:
-  disable: true
-
-steps:
- name: deploy
-  environment:
-    PASSWORD:
-      from_secret: HETZNER_TOKEN
-    DOMAIN:
-      from_secret: DOMAIN
-    CLOUDFLARE_TOKEN:
-      from_secret: CLOUDFLARE_TOKEN
-    USER_PASS:
-      from_secret: USER_PASS
-    INFECT_COMMIT_SHA: ${DRONE_COMMIT_SHA}
-
-  commands:
-  - set -o nounset
-  - > # TODO pass Base64 encoded password from Drone instead of this
-    ENCODED_PASSWORD="$(base64 <<<"$USER_PASS")"
-  # Create infect user script and then push it to a remote machine on server creation.
-  - |
-    cat << EOF > infect.sh
-    #! /usr/bin/env bash
-
-    set -o allexport
-    set -o errtrace
-    set -o pipefail
-    shopt -s inherit_errexit
-    trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR
-
-    API_TOKEN="$USER_PASS"
-    DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN
-    DNS_PROVIDER_TYPE=CLOUDFLARE
-    DOMAIN=$DOMAIN
-    ENCODED_PASSWORD="$ENCODED_PASSWORD"
-    HOSTNAME=selfprivacy-ci-test
-    LUSER=cicdcicd
-    NIXOS_CONFIG_ID=default
-    PROVIDER=HETZNER
-    SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K"
-    STAGING_ACME=true
-
-    curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \
-    | bash 2>&1 | tee /root/infect.log
-    EOF
-  - cat infect.sh
-  - location=fsn1
-  - set -o pipefail
-  - >
-    volume_id="$(http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" format=ext4 size:=10 location=$location name=10gb-by-drone-ci automount:=false | jq -re .volume.id)"
-  # TODO pass linux device volume_id name to infect.sh script
-  - >
-    http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 volumes:=[$volume_id] user_data=@infect.sh automount:=false location=$location
-
- name: dns
-  environment:
-    PASSWORD:
-      from_secret: HETZNER_TOKEN
-    CLOUDFLARE_TOKEN:
-      from_secret: CLOUDFLARE_TOKEN
-    DOMAIN:
-      from_secret: DOMAIN
-    ZONE_ID:
-      from_secret: ZONE_ID
-
-  commands:
-  # Read all available machines and export IP of 'ci-sibling'
-  # Create DNS entires for main domain and subdomains
-  - >
-    curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json
-  - >
-    export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' )
-  - echo $machineip
-  - >
-    curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN"
-    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
-    -H "Content-Type: application/json"
-  - >
-    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
-    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
-    -H "Content-Type: application/json"
-    --data '{"type":"A","name":"@","content":"'"$$machineip"'","ttl":3600,"priority":10,"proxied":false}'
-  - >
-    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=api content=$machineip ttl:=3600 priority:=10 proxied:=false
-  - >
-    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=cloud content=$machineip ttl:=3600 priority:=10 proxied:=false
-  - >
-    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=password content=$machineip ttl:=3600 priority:=10 proxied:=false
-  - >
-    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
-    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
-    -H "Content-Type: application/json"
-    --data '{"type":"MX","name":"@","content":"'"$$DOMAIN"'","ttl":3600,"priority":10,"proxied":false}'
-  - >
-    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
-    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
-    -H "Content-Type: application/json"
-    --data '{"type":"TXT","name":"_dmarc","content":"v=DMARC1; p=none","ttl":18000,"priority":10,"proxied":false}'
-  - >
-    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
-    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
-    -H "Content-Type: application/json"
-    --data "{\"type\":\"TXT\",\"name\":\"$DOMAIN\",\"content\":\"v=spf1 a mx ip4:$machineip -all\",\"ttl\":18000,\"priority\":10,\"proxied\":false}"
-
- name: wait for NixOS boot
-  environment:
-    DOMAIN:
-      from_secret: DOMAIN
-    TIMEOUT: 160
-  commands:
-  - sleep 300
-  - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done
-  - echo "give time for obtaining ACME certificate"
-  - sleep 60
-
- name: basic test
-  environment:
-    DOMAIN:
-      from_secret: DOMAIN
-    USER_PASS:
-      from_secret: USER_PASS
-    TEST_EMAIL_PASS:
-      from_secret: TEST_EMAIL_PASS
-    API_TOKEN:
-      from_secret: USER_PASS
-    TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git
-    TIMEOUT_1: 100
-  commands:
-  # ideally it should depend on $STAGING_ACME
-  # CERT_VERIFY=$(if [[ $STAGING_ACME == true ]]; then echo "--cacert root.pem"; else echo ""; fi)
-  - CERT_VERIFY="--cacert root.pem"
-  - source $(nix build $TESTS_REPO#library --print-out-paths)
-  - curl https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -sS --fail -o root.pem
-  - dig api.$DOMAIN
-  - >
-    retry $TIMEOUT_1 10 curl https://api.$DOMAIN/api/version -sS --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS"
-  - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp
-  - >
-    curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"'
-  - echo 'File contents to test nextcloud upload' > file
-  - >
-    curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -d @file -u "admin:$USER_PASS"
-  - >
-    curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS"
-
- name: API & nixos-rebuild test
-  environment:
-    DOMAIN:
-      from_secret: DOMAIN
-    API_TOKEN:
-      from_secret: USER_PASS
-    TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git
-    TIMEOUT_1: 120
-  commands:
-  - source $(nix build $TESTS_REPO#library --print-out-paths)
-  - retry $TIMEOUT_1 10 query-minimum-services
-  - enable-service bitwarden
-  - enable-service gitea
-  - enable-service jitsi-meet
-  - enable-service ocserv
-  - enable-service pleroma
-  - nixos-rebuild Rebuild
-  - sleep 180
-  - retry $TIMEOUT_1 10 query-all-services
-  - nixos-rebuild Rollback
-  - sleep 30
-  - retry $TIMEOUT_1 10 query-minimum-services
-  - enable-service bitwarden
-  - enable-service gitea
-  - enable-service jitsi-meet
-  - enable-service ocserv
-  - enable-service pleroma
-  - nixos-rebuild Rebuild
-  - sleep 60
-  - retry $TIMEOUT_1 10 query-all-services
-
- name: sleep after failure
-  environment:
-    DOMAIN:
-      from_secret: DOMAIN
-    TIMEOUT: 1500
-  commands:
-    - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done
-  when:
-    status:
-      - failure
-
- name: teardown
-  environment:
-    PASSWORD:
-      from_secret: HETZNER_TOKEN
-    CLOUDFLARE_TOKEN:
-      from_secret: CLOUDFLARE_TOKEN
-    DOMAIN:
-      from_secret: DOMAIN
-    ZONE_ID:
-      from_secret: ZONE_ID
-  commands:
-  # Delete our machine called 'ci-sibling' and all the found DNS records on it.
-  - >
-    curl -s "https://api.hetzner.cloud/v1/servers" -H "Authorization: Bearer $PASSWORD" > .hetzner_servers.json
-  - >
-    export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g')
-  - >
-    curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD"
-  - >
-    json=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json")
-  - >
-    echo "$json" > .dns-records.json
-  - >
-    result=$(jq -r ".result" <<< "$json")
-  - >
-    echo "$ZONE_ID"
-  - >
-    echo "$result" | jq '.[] | "\(.id)"' | while read id ; do echo "Deleting $id"; http -v --check-status --ignore-stdin DELETE https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$(echo $id | cut -f 2 -d '"') Authorization:"Bearer $CLOUDFLARE_TOKEN"; done
-  - location="fsn1"
-  - volume_name="10gb-by-drone-ci"
-  - >
-    volumes_json="$(http -d --print=HBb --ignore-stdin GET https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" name=$volume_name)"
-    # as of 2023-12-27 Hetzner ignores "name" query parameter (and others too)!!
-  - >
-    volume_id="$(jq <<<"$volumes_json" -re "(.volumes[] | select(.name == \"$volume_name\")).id")"
-  - sleep 1
-  - >
-    http -d --print=HBb --ignore-stdin DELETE https://api.hetzner.cloud/v1/volumes/$volume_id Authorization:"Bearer $PASSWORD"
-  when:
-    status:
-    - failure
-    - success
+local myVar = std.extVar("input.myVar");
+{
+  "kind": "pipeline",
+  "type": "exec",
+  "name": "default",
+  "clone": { "disable": true },
+  "steps": [
+    {
+      "name": "step1",
+      "commands": [
+        myVar
+      ]
+    }
+  ]
+}
--- a/129
+++ b/129
@ -4,24 +4,20 @@
 # - https://github.com/elitak/nixos-infect
 # - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect

-# mandatory variables
 : "${API_TOKEN:?API_TOKEN variable is not set}"
-: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}"
-: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}"
 : "${DOMAIN:?DOMAIN variable is not set}"
-: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}"
-: "${HOSTNAME:?HOSTNAME variable is not set}"
 : "${LUSER:?LUSER variable is not set}"
+: "${HOSTNAME:?HOSTNAME variable is not set}"
 : "${PROVIDER:?PROVIDER variable is not set}"
-
-# optional variables
-: "${NIXOS_CONFIG_ID:=default}"
+: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}"
+: "${STAGING_ACME:?STAGING_ACME variable is not set}"
+: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}"
+: "${DB_PASSWORD:?DB_PASSWORD variable is not set}"
+: "${USER_PASS:?USER_PASS variable is not set}"
+: "${NIX_VERSION:?NIX_VERSION variable is not set}"
+: "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}"
+: "${CONFIG_URL:?CONFIG_URL variable is not set}"
 : "${SSH_AUTHORIZED_KEY:=}"
-: "${VOLUME_DEV_PATH:=}"
-: "${STAGING_ACME:=false}"
-
-readonly CONFIG_URL="https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/c5b16ac16dd328199e8f4f901890fff63f2fb81f.tar.gz"
-readonly NIX_VERSION="2.18.1"

 readonly NL=$'\n'
 readonly LOCAL_FLAKE_DIR="/etc/nixos"
@ -31,41 +27,15 @@ readonly NIX_OPTS=(--extra-experimental-features "nix-command flakes")
 DoNetConf=

 genOptionalSsh() {
-  if [ -n "${SSH_AUTHORIZED_KEY}" ]; then
-      cat << EOF
+  [ -n "${SSH_AUTHORIZED_KEY}" ] && cat << EOF
 "ssh": { "rootKeys": [ "${SSH_AUTHORIZED_KEY}" ] },
 EOF
-  fi
-}
-
-# TODO receive disk device path from outside
-determine2ndDisk() {
-  local volDevPath rootFsDev
-  if [ -n "$VOLUME_DEV_PATH" ]; then
-      volDevPath="$VOLUME_DEV_PATH"
-  else
-      case "$PROVIDER" in
-          HETZNER)
-              volDevPath="/dev/sdb"
-              ;;
-          DIGITALOCEAN)
-              volDevPath="/dev/sda"
-              ;;
-          *)
-              return 1
-              ;;
-      esac
-  fi
-  rootFsDev="$(findmnt -no source -T /)"
-  [[ -b "$volDevPath" && "$volDevPath" != "$rootFsDev" ]] && echo "$volDevPath"
 }

 # Merge original userdata.json with deployment specific fields and print result.
 genUserdata() {
-  local hashed_password diskDeviceName userdata_infect
-  hashed_password="$(mkpasswd -m sha-512 "$USER_PASS")"
-  diskDevice="$(determine2ndDisk)"
-  diskDeviceName="${diskDevice##/dev/}"
+  local HASHED_PASSWORD userdata_infect release
+  HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")"

  userdata_infect=$(cat << EOF
 {
@ -78,33 +48,9 @@ genUserdata() {
        "provider": "$PROVIDER"
    },
    "domain": "$DOMAIN",
-    "hashedMasterPassword": "$hashed_password",
+    "hashedMasterPassword": "$HASHED_PASSWORD",
    "hostname": "$HOSTNAME",
-    "username": "$LUSER",
-    "volumes": [
-        {
-            "device": "$diskDevice",
-            "mountPoint": "/volumes/$diskDeviceName",
-            "fsType": "ext4"
-        }
-    ],
-    "modules": {
-        "bitwarden": {
-            "location": "$diskDeviceName"
-        },
-        "gitea": {
-            "location": "$diskDeviceName"
-        },
-        "nextcloud": {
-            "location": "$diskDeviceName"
-        },
-        "pleroma": {
-            "location": "$diskDeviceName"
-        },
-        "simple-nixos-mailserver": {
-            "location": "$diskDeviceName"
-        }
-    }
+    "username": "$LUSER"
 }
 EOF
 )
@ -114,16 +60,13 @@ EOF
 }

 genSecrets() {
-  local dbpass
-  dbpass="$(shuf --random-source=/dev/urandom -erz -n32 {A..Z} {a..z} {0..9} | tr -d '\n')"
-
  cat << EOF
 {
    "api": {
        "token": "$API_TOKEN",
        "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"]
    },
-    "databasePassword": "$dbpass",
+    "databasePassword": "$DB_PASSWORD",
    "dns": {
        "apiKey": "$DNS_PROVIDER_TOKEN"
    },
@ -166,7 +109,7 @@ EOF
  # FIXME it's questionable whether these modules are needed at all...
  declare -a availableKernelModules=()

-  [ "$PROVIDER" == "DIGITALOCEAN" ] \
+  [ "$PROVIDER" == "digitalocean" ] \
  && availableKernelModules+=('"ata_piix"' '"uhci_hcd"' '"xen_blkfront"')

  [ "$(uname -m)" == "x86_64" ] \
@ -210,7 +153,7 @@ EOF

 setupConf() {
  mkdir -p ${LOCAL_FLAKE_DIR}
-  if ! curl --fail "${CONFIG_URL}" \
+  if ! curl "${CONFIG_URL}" \
  | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 --exclude=".*"
  then
      echo "Error downloading/extracting top level flake configuration!"
@ -350,12 +293,6 @@ findESP() {
 }

 prepareEnv() {
-  if ! USER_PASS="$(base64 -d <<<"$ENCODED_PASSWORD")"; then
-      echo "Error decoding ENCODED_PASSWORD from Base64!"
-      exit 1
-  fi
-  readonly USER_PASS
-
  isEFI=0
  [ -d /sys/firmware/efi ] && isEFI=1

@ -431,8 +368,7 @@ checkEnv() {
  req xzcat            || { echo "ERROR: Missing xzcat";               return 1; }
  req awk              || { echo "ERROR: Missing awk";                 return 1; }
  req cut || req df    || { echo "ERROR: Missing coreutils (cut, df)"; return 1; }
-  req mkpasswd         || { echo "ERROR: Missing mkpasswd";            return 1; }
-  req shuf             || { echo "ERROR: Missing shuf";                return 1; }
+  req mkpasswd         || { echo "ERROR: Missing mkpasswd"; return 1; }
 }

 # Download and execute the nix installer script.
@ -453,12 +389,12 @@ installNix() {
  fi

  echo "Downloading install script from ${installURL}..."
-  if ! curl --fail "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then
+  if ! curl "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then
      echo "Failure while downloading Nix install script!"
      return 1
  fi

-  if ! sha="$(curl --fail "${shaURL}")"; then
+  if ! sha="$(curl "${shaURL}")"; then
      echo "Failure while downloading Nix install script sha!"
      return 1
  fi
@ -481,16 +417,17 @@ installNix() {
 }

 infect() {
-  # this is needed solely for accepting the sp-module sub-flake
+  # this is needed solely for accepting the sp-module subflake
  # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478
-  cd /etc/nixos || return 1
-  $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} \
-  --override-input sp-modules path:./sp-modules
+  /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \
+  --extra-experimental-features "nix-command flakes" \
+  --override-input sp-modules ${LOCAL_FLAKE_DIR}/sp-modules

  echo "nix build the configuration flake..."
-  if ! $NIX "${NIX_OPTS[@]}" build \
+  if ! /root/.nix-profile/bin/nix build \
+  --extra-experimental-features "nix-command flakes" \
  --profile /nix/var/nix/profiles/system \
-  ${LOCAL_FLAKE_DIR}/#nixosConfigurations."$NIXOS_CONFIG_ID".config.system.build.toplevel
+  ${LOCAL_FLAKE_DIR}/#nixosConfigurations.sp-nixos.config.system.build.toplevel
  then
      echo "Failed!"
      exit 1
@ -520,20 +457,20 @@ infect() {
  fi

  # Remove nix installed by the "install" script.
-  rm -fv /nix/var/nix/profiles/default* /root/.nix-profile
-  rm -rv /nix/var/nix/profiles/per-user
+  rm -fv /nix/var/nix/profiles/default*
  /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage
 }

-set -o errtrace
-set -o nounset
 set -o pipefail
+set -o nounset
+set -o errexit
 set -o xtrace
 shopt -s inherit_errexit
-trap 'echo ${LINENO}: "$BASH_COMMAND"; exit 1' ERR
+
+genNetworkingConf

 # digitalocean requires detailed network config to be generated
-[ "$PROVIDER" == "DIGITALOCEAN" ] && DoNetConf="y"
+[ "$PROVIDER" == "digitalocean" ] && DoNetConf="y"

 apt update
 apt install -y git tar curl whois jq