From eca67809838944ad8b2feb9cc946ae37418e6e6a Mon Sep 17 00:00:00 2001 From: inexcode Date: Mon, 10 Oct 2022 17:40:46 +0300 Subject: [PATCH 001/124] feat: Initialize server with binds and resizable volume activated --- nixos-infect | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/nixos-infect b/nixos-infect index 349eed6..a9d7ef6 100755 --- a/nixos-infect +++ b/nixos-infect @@ -10,7 +10,7 @@ makeConf() { if [[ $PASSWORD == null ]]; then export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode) fi - + export ESCAPED_PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode | jq -Rs .) export HASHED_PASSWORD=$( mkpasswd -m sha-512 "$PASSWORD" ) @@ -31,7 +31,8 @@ makeConf() { cat > /etc/nixos/userdata/userdata.json << EOF { "api": { - "token": "$API_TOKEN" + "token": "$API_TOKEN", + "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"] }, "backblaze": { "accountId": "$BACKBLAZE_KEY_ID", @@ -39,7 +40,8 @@ makeConf() { "bucket": "$BACKBLAZE_BUCKET_NAME" }, "bitwarden": { - "enable": true + "enable": true, + "location": "sdb" }, "cloudflare": { "apiKey": "$CF_TOKEN" @@ -51,23 +53,34 @@ makeConf() { "nextcloud": { "enable": true, "adminPassword": $ESCAPED_PASSWORD, - "databasePassword": $ESCAPED_PASSWORD + "databasePassword": $ESCAPED_PASSWORD, + "location": "sdb" }, "gitea": { - "enable": true + "enable": true, + "location": "sdb" }, "jitsi": { "enable": true }, "ocserv": { - "enable": true + "enable": false }, "pleroma": { - "enable": true + "enable": false, + "location": "sdb" }, "timezone": "Europe/Uzhgorod", "resticPassword": $ESCAPED_PASSWORD, - "username": "$LUSER" + "username": "$LUSER", + "volumes": [ + { + "device": "/dev/sdb", + "mountPoint": "/volumes/sdb", + "fsType": "ext4" + } + ], + "useBinds": true } EOF chmod 0600 /etc/nixos/userdata/userdata.json @@ -353,4 +366,4 @@ removeSwap if [[ -z "$NO_REBOOT" ]]; then reboot -fi \ No newline at end of file +fi From bb59d623c5d981af20d3628272b3ed6a9b010fdd Mon Sep 17 00:00:00 2001 From: NaiJi Date: Wed, 16 Nov 2022 03:39:34 +0400 Subject: [PATCH 002/124] chore!: Change to temp branch for test DON'T MERGE --- nixos-infect | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index a9d7ef6..5363082 100755 --- a/nixos-infect +++ b/nixos-infect @@ -16,8 +16,9 @@ makeConf() { # NB <<"EOF" quotes / $ ` in heredocs, < Date: Wed, 16 Nov 2022 11:06:15 +0300 Subject: [PATCH 003/124] feat: Add support for staging ACME --- nixos-infect | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nixos-infect b/nixos-infect index 5363082..9cc91bb 100755 --- a/nixos-infect +++ b/nixos-infect @@ -11,12 +11,16 @@ makeConf() { export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode) fi + if [[ $STAGING_ACME == null ]]; then + export STAGING_ACME='false' + fi + export ESCAPED_PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode | jq -Rs .) export HASHED_PASSWORD=$( mkpasswd -m sha-512 "$PASSWORD" ) # NB <<"EOF" quotes / $ ` in heredocs, < Date: Wed, 16 Nov 2022 11:11:22 +0300 Subject: [PATCH 004/124] fix: JSON formatting and config branch --- nixos-infect | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index 9cc91bb..31b319a 100755 --- a/nixos-infect +++ b/nixos-infect @@ -21,7 +21,9 @@ makeConf() { # NB <<"EOF" quotes / $ ` in heredocs, < Date: Thu, 17 Nov 2022 11:19:16 +0300 Subject: [PATCH 005/124] fix: userdata.json formatting --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 31b319a..3ada00f 100755 --- a/nixos-infect +++ b/nixos-infect @@ -57,7 +57,7 @@ makeConf() { "useStagingACME": $STAGING_ACME }, "server": { - "provider": "HETZNER", + "provider": "HETZNER" }, "databasePassword": "$DB_PASSWORD", "domain": "$DOMAIN", From be62d1913e98526aece15272f9e5ed74722fe3a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?NaiJi=20=E2=9C=A8?= Date: Fri, 18 Nov 2022 07:54:32 +0200 Subject: [PATCH 006/124] chore!: Change to test branch Please don't forget to overwrite with api-redis again --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 3ada00f..dab4779 100755 --- a/nixos-infect +++ b/nixos-infect @@ -22,7 +22,7 @@ makeConf() { mkdir -p /etc/nixos - git clone -b api-redis https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos + git clone -b pleasedontmerge https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos # git clone https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos From d88d75b9b7be7923a4efc940eb881b59e818a0b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?NaiJi=20=E2=9C=A8?= Date: Sun, 20 Nov 2022 16:39:01 +0200 Subject: [PATCH 007/124] chore: Remove temporary cloning branch and change back to api-redis Testing is over :') now back to normal --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index dab4779..3ada00f 100755 --- a/nixos-infect +++ b/nixos-infect @@ -22,7 +22,7 @@ makeConf() { mkdir -p /etc/nixos - git clone -b pleasedontmerge https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos + git clone -b api-redis https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos # git clone https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos From f0741188b15e968691999410c97f343907deb2e0 Mon Sep 17 00:00:00 2001 From: NaiJi Date: Tue, 17 Jan 2023 17:12:53 +0400 Subject: [PATCH 008/124] feat: Implement dns provider type variable for infecting --- nixos-infect | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 3ada00f..4325c77 100755 --- a/nixos-infect +++ b/nixos-infect @@ -11,6 +11,10 @@ makeConf() { export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode) fi + if [[ $DNS_PROVIDER_TYPE == null ]]; then + export DNS_PROVIDER_TYPE='CLOUDFLARE' + fi + if [[ $STAGING_ACME == null ]]; then export STAGING_ACME='false' fi @@ -52,7 +56,7 @@ makeConf() { "location": "sdb" }, "dns": { - "provider": "CLOUDFLARE", + "provider": "$DNS_PROVIDER_TYPE", "apiKey": "$CF_TOKEN", "useStagingACME": $STAGING_ACME }, From 3fda846808e9e152ff2484df907dcaa07e79a813 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Wed, 8 Feb 2023 16:27:50 +0300 Subject: [PATCH 009/124] fix: Defaults not being set --- nixos-infect | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos-infect b/nixos-infect index 4325c77..ee90527 100755 --- a/nixos-infect +++ b/nixos-infect @@ -7,15 +7,15 @@ set -e -o pipefail makeConf() { # Skip everything if main config already present [[ -e /etc/nixos/configuration.nix ]] && return 0 - if [[ $PASSWORD == null ]]; then + if [[ -z $PASSWORD ]]; then export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode) fi - if [[ $DNS_PROVIDER_TYPE == null ]]; then + if [[ -z $DNS_PROVIDER_TYPE ]]; then export DNS_PROVIDER_TYPE='CLOUDFLARE' fi - if [[ $STAGING_ACME == null ]]; then + if [[ -z $STAGING_ACME ]]; then export STAGING_ACME='false' fi From 491f836dc64d5bc990fc09008377ef0a56f04d2c Mon Sep 17 00:00:00 2001 From: Inex Code Date: Fri, 17 Mar 2023 14:08:23 +0300 Subject: [PATCH 010/124] chore: Use master branch of NixOS config --- nixos-infect | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nixos-infect b/nixos-infect index ee90527..ba285ec 100755 --- a/nixos-infect +++ b/nixos-infect @@ -25,10 +25,7 @@ makeConf() { # NB <<"EOF" quotes / $ ` in heredocs, < Date: Sat, 18 Nov 2023 06:47:34 +0400 Subject: [PATCH 011/124] first test --- .drone.yml | 146 ++++++++++++++++++++ nixos-infect | 378 ++++++++++++++++++++++++--------------------------- 2 files changed, 325 insertions(+), 199 deletions(-) create mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..2eefbc7 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,146 @@ +kind: pipeline +type: exec +name: default + +steps: +- name: deploy + environment: + PASSWORD: + from_secret: HETZNER_TOKEN + DOMAIN: + from_secret: DOMAIN + CLOUDFLARE_TOKEN: + from_secret: CLOUDFLARE_TOKEN + USER_PASS: + from_secret: USER_PASS + + commands: + # Create infect user script and then export it to a remote machine on server creation. + - echo '#!/bin/sh' > infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - echo "Wait for 100 seconds." + - sleep 100 + - false + +- name: dns + environment: + PASSWORD: + from_secret: HETZNER_TOKEN + CLOUDFLARE_TOKEN: + from_secret: CLOUDFLARE_TOKEN + DOMAIN: + from_secret: DOMAIN + ZONE_ID: + from_secret: ZONE_ID + + commands: + # Read all available machines and export IP of 'ci-sibling' + # Create DNS entires for main domain and subdomains + - > + curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json + - > + export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' ) + - echo $machineip + - > + curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN" + -H "Authorization: Bearer $CLOUDFLARE_TOKEN" + -H "Content-Type: application/json" + - > + curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" + -H "Authorization: Bearer $CLOUDFLARE_TOKEN" + -H "Content-Type: application/json" + --data '{"type":"A","name":"@","content":"'"$$machineip"'","ttl":3600,"priority":10,"proxied":false}' + - > + http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=api content=$machineip ttl:=3600 priority:=10 proxied:=false + - > + http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=cloud content=$machineip ttl:=3600 priority:=10 proxied:=false + - > + http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=password content=$machineip ttl:=3600 priority:=10 proxied:=false + - > + curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" + -H "Authorization: Bearer $CLOUDFLARE_TOKEN" + -H "Content-Type: application/json" + --data '{"type":"MX","name":"@","content":"'"$$DOMAIN"'","ttl":3600,"priority":10,"proxied":false}' + - > + curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" + -H "Authorization: Bearer $CLOUDFLARE_TOKEN" + -H "Content-Type: application/json" + --data '{"type":"TXT","name":"_dmarc","content":"v=DMARC1; p=none","ttl":18000,"priority":10,"proxied":false}' + - > + curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" + -H "Authorization: Bearer $CLOUDFLARE_TOKEN" + -H "Content-Type: application/json" + --data '{"type":"TXT","name":"'"$$DOMAIN"'","content":"v=spf1 a mx ip4:$machineip -all","ttl":18000,"priority":10,"proxied":false}' + +- name: sleep + commands: + - echo "Wait for 100 seconds." + - sleep 100 + +- name: test + environment: + DOMAIN: + from_secret: DOMAIN + USER_PASS: + from_secret: USER_PASS + TEST_EMAIL_PASS: + from_secret: TEST_EMAIL_PASS + commands: + - > + dig api.$DOMAIN + - > + http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - > + swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS + - > + http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - > + echo "File to test nextcloud upload" > file + - > + http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS + - > + http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS + #- > + #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics + +- name: teardown + environment: + PASSWORD: + from_secret: HETZNER_TOKEN + CLOUDFLARE_TOKEN: + from_secret: CLOUDFLARE_TOKEN + DOMAIN: + from_secret: DOMAIN + ZONE_ID: + from_secret: ZONE_ID + commands: + # Delete our machine called 'ci-sibling' and all the found DNS records on it. + - > + curl -s "https://api.hetzner.cloud/v1/servers" -H "Authorization: Bearer $PASSWORD" > .hetzner_servers.json + - > + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + - > + curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" + - > + json=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json") + - > + echo "$json" > .dns-records.json + - > + result=$(jq -r ".result" <<< "$json") + - > + echo "$ZONE_ID" + - > + echo "$result" | jq '.[] | "\(.id)"' | while read id ; do echo "Deleting $id"; http -v --check-status --ignore-stdin DELETE https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$(echo $id | cut -f 2 -d '"') Authorization:"Bearer $CLOUDFLARE_TOKEN"; done + when: + status: + - failure + - success + +- name: switch + environment: + CHANNEL_SWITCHER_KEY: + from_secret: CHANNEL_SWITCHER_KEY + commands: + - > + curl -X POST -H "Authorization: $CHANNEL_SWITCHER_KEY" 'https://channel.selfprivacy.org/nixos-selfpricacy/upgrade' diff --git a/nixos-infect b/nixos-infect index ba285ec..1c2878f 100755 --- a/nixos-infect +++ b/nixos-infect @@ -1,205 +1,129 @@ #! /usr/bin/env bash -# More info at: https://github.com/elitak/nixos-infect +# More info at: +# - https://github.com/elitak/nixos-infect +# - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect -set -e -o pipefail +: "${API_TOKEN:?API_TOKEN variable is not set}" +: "${DOMAIN:?DOMAIN variable is not set}" +: "${LUSER:?LUSER variable is not set}" +: "${HOSTNAME:?HOSTNAME variable is not set}" +: "${PROVIDER:?PROVIDER variable is not set}" +: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}" +: "${STAGING_ACME:?STAGING_ACME variable is not set}" +: "${CF_TOKEN:?CF_TOKEN variable is not set}" +: "${DB_PASSWORD:?DB_PASSWORD variable is not set}" +: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" +: "${NIX_VERSION:?NIX_VERSION variable is not set}" +: "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}" +: "${SUDO_USER:?SUDO_USER variable is not set}" +: "${CONFIG_URL:?CONFIG_URL variable is not set}" -makeConf() { - # Skip everything if main config already present - [[ -e /etc/nixos/configuration.nix ]] && return 0 - if [[ -z $PASSWORD ]]; then - export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode) - fi +readonly LOCAL_FLAKE_DIR="/etc/nixos" +readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json" - if [[ -z $DNS_PROVIDER_TYPE ]]; then - export DNS_PROVIDER_TYPE='CLOUDFLARE' - fi +# Merge original userdata.json with deployment specific fields and print result. +genUserdata() { + local password HASHED_PASSWORD userdata_infect + password=$(printf "%s" "$ENCODED_PASSWORD" | base64 --decode) + HASHED_PASSWORD=$(mkpasswd -m sha-512 "$password") - if [[ -z $STAGING_ACME ]]; then - export STAGING_ACME='false' - fi + userdata_infect=$(cat << EOF +{ + "dns": { + "provider": "$DNS_PROVIDER_TYPE", + "useStagingACME": "$STAGING_ACME" + }, + "server": { + "provider": "$PROVIDER" + }, + "domain": "DOMAIN", + "hashedMasterPassword": "$HASHED_PASSWORD", + "hostname": "HOSTNAME", + "username": "$LADMIN" +} +EOF +) - export ESCAPED_PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode | jq -Rs .) - export HASHED_PASSWORD=$( mkpasswd -m sha-512 "$PASSWORD" ) + jq -s '.[0] * .[1]' \ + "${1:?no userdata.json given to merge with}" <(printf "%s" "$userdata_infect") +} - # NB <<"EOF" quotes / $ ` in heredocs, < /etc/nixos/userdata/userdata.json << EOF + cat << EOF { "api": { "token": "$API_TOKEN", "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"] }, - "backup": { - "provider": "BACKBLAZE", - "accountId": "$BACKBLAZE_KEY_ID", - "accountKey": "$BACKBLAZE_ACCOUNT_KEY", - "bucket": "$BACKBLAZE_BUCKET_NAME" - }, - "bitwarden": { - "enable": true, - "location": "sdb" - }, - "dns": { - "provider": "$DNS_PROVIDER_TYPE", - "apiKey": "$CF_TOKEN", - "useStagingACME": $STAGING_ACME - }, - "server": { - "provider": "HETZNER" - }, "databasePassword": "$DB_PASSWORD", - "domain": "$DOMAIN", - "hashedMasterPassword": "$HASHED_PASSWORD", - "hostname": "$HOSTNAME", - "nextcloud": { - "enable": true, - "adminPassword": $ESCAPED_PASSWORD, - "databasePassword": $ESCAPED_PASSWORD, - "location": "sdb" + "dns": { + "apiKey": "$CF_TOKEN" }, - "gitea": { - "enable": true, - "location": "sdb" + "modules": { + "nextcloud": { + "adminPassword": $ESCAPED_PASSWORD, + "databasePassword": $ESCAPED_PASSWORD, + } }, - "jitsi": { - "enable": true - }, - "ocserv": { - "enable": false - }, - "pleroma": { - "enable": false, - "location": "sdb" - }, - "timezone": "Europe/Uzhgorod", - "resticPassword": $ESCAPED_PASSWORD, - "username": "$LUSER", - "volumes": [ - { - "device": "/dev/sdb", - "mountPoint": "/volumes/sdb", - "fsType": "ext4" - } - ], - "useBinds": true + "resticPassword": $ESCAPED_PASSWORD } EOF - chmod 0600 /etc/nixos/userdata/userdata.json +} - if isEFI; then +genHardwareConfiguration() { + local bootcfg + if ((isEFI)); then bootcfg=$(cat << EOF boot.loader.grub = { efiSupport = true; efiInstallAsRemovable = true; device = "nodev"; }; - fileSystems."/boot" = { device = "$esp"; fsType = "vfat"; }; + fileSystems."/boot" = { device = "$ESP"; fsType = "vfat"; }; EOF ) else bootcfg=$(cat << EOF - boot.loader.grub.device = "$grubdev"; + boot.loader.grub.device = "$GRUBDEV"; EOF ) fi - # If you rerun this later, be sure to prune the filesSystems attr - cat > /etc/nixos/hardware-configuration.nix << EOF + cat << EOF { modulesPath, ... }: { imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; $bootcfg boot.initrd.kernelModules = [ "nvme" ]; - fileSystems."/" = { device = "$rootfsdev"; fsType = "$rootfstype"; }; + fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; }; } EOF - - [[ -n "$doNetConf" ]] && makeNetworkingConf || true } -makeNetworkingConf() { - # XXX It'd be better if we used procfs for all this... - local IFS=$'\n' - eth0_name=$(ip address show | grep '^2:' | awk -F': ' '{print $2}') - eth0_ip4s=$(ip address show dev "$eth0_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|') - eth0_ip6s=$(ip address show dev "$eth0_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|' || '') - gateway=$(ip route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9.]+).*|\1|') - gateway6=$(ip -6 route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|' || true) - ether0=$(ip address show dev "$eth0_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|') - - eth1_name=$(ip address show | grep '^3:' | awk -F': ' '{print $2}')||true - if [ -n "$eth1_name" ];then - eth1_ip4s=$(ip address show dev "$eth1_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|') - eth1_ip6s=$(ip address show dev "$eth1_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|' || '') - ether1=$(ip address show dev "$eth1_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|') - interfaces1=<< EOF - $eth1_name = { - ipv4.addresses = [$(for a in "${eth1_ip4s[@]}"; do echo -n " - $a"; done) - ]; - ipv6.addresses = [$(for a in "${eth1_ip6s[@]}"; do echo -n " - $a"; done) - ]; -EOF - extraRules1="ATTR{address}==\"${ether1}\", NAME=\"${eth1_name}\"" - else - interfaces1="" - extraRules1="" +setupConf() { + mkdir -p ${LOCAL_FLAKE_DIR} + if ! curl "${CONFIG_URL}" | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 + then + echo "Error downloading/extracting top level flake configuration!" + exit 1 fi - readarray nameservers < <(grep ^nameserver /etc/resolv.conf | sed -r \ - -e 's/^nameserver[[:space:]]+([0-9.a-fA-F:]+).*/"\1"/' \ - -e 's/127[0-9.]+/8.8.8.8/' \ - -e 's/::1/8.8.8.8/' ) + # generate and write hardware-configuration.nix + genHardwareConfiguration > ${LOCAL_FLAKE_DIR}/hardware-configuration.nix - if [[ "$eth0_name" = eth* ]]; then - predictable_inames="usePredictableInterfaceNames = lib.mkForce false;" - else - predictable_inames="usePredictableInterfaceNames = lib.mkForce true;" - fi - cat > /etc/nixos/networking.nix << EOF -{ lib, ... }: { - # This file was populated at runtime with the networking - # details gathered from the active system. - networking = { - nameservers = [ ${nameservers[@]} ]; - defaultGateway = "${gateway}"; - defaultGateway6 = "${gateway6}"; - dhcpcd.enable = false; - $predictable_inames - interfaces = { - $eth0_name = { - ipv4.addresses = [$(for a in "${eth0_ip4s[@]}"; do echo -n " - $a"; done) - ]; - ipv6.addresses = [$(for a in "${eth0_ip6s[@]}"; do echo -n " - $a"; done) - ]; - ipv4.routes = [ { address = "${gateway}"; prefixLength = 32; } ]; - ipv6.routes = [ { address = "${gateway6}"; prefixLength = 128; } ]; - }; - $interfaces1 - }; - }; - services.udev.extraRules = '' - ATTR{address}=="${ether0}", NAME="${eth0_name}" - $extraRules1 - ''; -} -EOF + # generate infected userdata based on original + local userdataInfected + userdataInfected="$(genUserdata ${LOCAL_FLAKE_DIR}/userdata.json)" + printf "%s" "$userdataInfected" > ${LOCAL_FLAKE_DIR}/userdata.json + + # generate and write secrets + local secrets + secrets="$(genSecrets)" + install -m0600 <(printf "%s" "$secrets") -DT ${SECRETS_FILEPATH} } makeSwap() { @@ -216,12 +140,8 @@ removeSwap() { rm -vf /tmp/nixos-infect.*.swp } -isEFI() { - [ -d /sys/firmware/efi ] -} - findESP() { - esp="" + local esp for d in /boot/EFI /boot/efi /boot; do [[ ! -d "$d" ]] && continue [[ "$d" == "$(df "$d" --output=target | sed 1d)" ]] \ @@ -230,22 +150,26 @@ findESP() { done [[ -z "$esp" ]] && { echo "ERROR: No ESP mount point found"; return 1; } for uuid in /dev/disk/by-uuid/*; do - [[ $(readlink -f "$uuid") == "$esp" ]] && echo $uuid && return 0 + [[ $(readlink -f "$uuid") == "$esp" ]] && echo "$uuid" && return 0 done } prepareEnv() { - # $esp and $grubdev are used in makeConf() - if isEFI; then - esp="$(findESP)" + isEFI=0 + [ -d /sys/firmware/efi ] || isEFI=1 + + if ((isEFI)); then + ESP="$(findESP)" else - for grubdev in /dev/vda /dev/sda /dev/nvme0n1 ; do [[ -e $grubdev ]] && break; done + for GRUBDEV in /dev/vda /dev/sda /dev/nvme0n1; do + [[ -e $GRUBDEV ]] && break; + done fi # Retrieve root fs block device # (get root mount) (get partition or logical volume) - rootfsdev=$(mount | grep "on / type" | awk '{print $1;}') - rootfstype=$(df $rootfsdev --output=fstype | sed 1d) + ROOTFSDEV=$(mount | grep "on / type" | awk '{print $1;}') + ROOTFSTYPE=$(df "$ROOTFSDEV" --output=fstype | sed 1d) # DigitalOcean doesn't seem to set USER while running user data export USER="root" @@ -295,9 +219,6 @@ checkEnv() { # Perform some easy fixups before checking # TODO prevent multiple calls to apt-get update (which dnf && dnf install -y perl-Digest-SHA) || true # Fedora 24 - which bzcat || (which yum && yum install -y bzip2) \ - || (which apt-get && apt-get update && apt-get install -y bzip2) \ - || true which xzcat || (which yum && yum install -y xz-utils) \ || (which apt-get && apt-get update && apt-get install -y xz-utils) \ || true @@ -306,48 +227,100 @@ checkEnv() { || true req curl || req wget || { echo "ERROR: Missing both curl and wget"; return 1; } - req bzcat || { echo "ERROR: Missing bzcat"; return 1; } req xzcat || { echo "ERROR: Missing xzcat"; return 1; } - req groupadd || { echo "ERROR: Missing groupadd"; return 1; } - req useradd || { echo "ERROR: Missing useradd"; return 1; } - req ip || { echo "ERROR: Missing ip"; return 1; } req awk || { echo "ERROR: Missing awk"; return 1; } req cut || req df || { echo "ERROR: Missing coreutils (cut, df)"; return 1; } } +# Download and execute the nix installer script. +installNix() { + local nixReleaseBase='https://releases.nixos.org' + local installURL="${nixReleaseBase}/nix/nix-${NIX_VERSION}/install" + local shaURL="${installURL}.sha256" + local sha tmpNixInstall + + # temporary destination for install script + tmpNixInstall="$(mktemp -t nix-install-XXXXXXXXXX)" + if [[ ! -f "${tmpNixInstall}" ]]; then + echo "Failed creating a temporary file for Nix install script!" + return 1 + fi + + echo "Downloading install script from ${installURL}..." + if ! curl "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then + echo "Failure while downloading Nix install script!" + return 1 + fi + + if ! sha="$(curl "${shaURL}")"; then + echo "Failure while downloading Nix install script sha!" + return 1 + fi + + echo "Validating Nix install script checksum..." + if ! echo "${sha} ${tmpNixInstall}" | sha256sum -c; then + echo "Checksum validation failed!" + return 1 + fi + + echo "Running nix installer..." + if $SHELL "${tmpNixInstall}"; then + echo "Nix is installed" + rm "${tmpNixInstall}" + else + echo "Nix installation script failed!" + return 1 + fi +} + infect() { # Add nix build users # FIXME run only if necessary, rather than defaulting true - groupadd nixbld -g 30000 || true - for i in {1..10}; do - useradd -c "Nix build user $i" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" "nixbld$i" || true - done + # groupadd nixbld -g 30000 || true + # for i in {1..10}; do + # useradd -c "Nix build user $i" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" "nixbld$i" || true + # done # TODO use addgroup and adduser as fallbacks #addgroup nixbld -g 30000 || true #for i in {1..10}; do adduser -DH -G nixbld nixbld$i || true; done - curl -L https://nixos.org/nix/install | $SHELL + # install Nix in the current system + # (this should create system nixbld users too) + # curl -L "https://releases.nixos.org/nix/nix-$NIX_VERSION/install" | $SHELL + if ! installNix; then + echo "Nix installation failed!" + exit 1 + fi # shellcheck disable=SC1090 - source ~/.nix-profile/etc/profile.d/nix.sh + # is it needed? + # source ~/.nix-profile/etc/profile.d/nix.sh - nix-channel --remove nixpkgs - nix-channel --add "https://channel.selfprivacy.org/nixos-selfpricacy" nixos - nix-channel --update + echo "nix build the configuration flake..." + if ! /root/.nix-profile/bin/nix build \ + --extra-experimental-features "nix-command flakes" \ + --profile /nix/var/nix/profiles/system \ + ${LOCAL_FLAKE_DIR}/#nixosConfigurations.sp-nixos.config.system.build.toplevel + then + echo "Failed!" + exit 1 + fi - export NIXOS_CONFIG=/etc/nixos/configuration.nix + echo "activate the built configuration..." + if ! /nix/var/nix/profiles/system/activate; then + echo "Failed!"; exit 1 + fi - nix-env --set \ - -I nixpkgs=$HOME/.nix-defexpr/channels/nixos \ - -f '' \ - -p /nix/var/nix/profiles/system \ - -A system + echo "make configuration switch..." + if /nix/var/nix/profiles/system/bin/switch-to-configuration switch; then + echo "Failed!"; exit 1 + fi - # Remove nix installed with curl | bash + # Remove nix installed by the "install" script. rm -fv /nix/var/nix/profiles/default* /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage - # Reify resolv.conf + # Reify resolv.conf (???) [[ -L /etc/resolv.conf ]] && mv -v /etc/resolv.conf /etc/resolv.conf.lnk && cat /etc/resolv.conf.lnk > /etc/resolv.conf # Stage the Nix coup d'état @@ -357,24 +330,31 @@ infect() { echo root/.nix-defexpr/channels >> /etc/NIXOS_LUSTRATE rm -rf /boot.bak - isEFI && umount "$esp" + ((isEFI)) && umount "$ESP" mv -v /boot /boot.bak - if isEFI; then + if ((isEFI)); then mkdir /boot - mount "$esp" /boot + mount "$ESP" /boot find /boot -depth ! -path /boot -exec rm -rf {} + fi - /nix/var/nix/profiles/system/bin/switch-to-configuration boot + + echo "make configuration boot by default..." + if /nix/var/nix/profiles/system/bin/switch-to-configuration boot; then + echo "Failed!"; exit 1 + fi } -[ "$PROVIDER" = "digitalocean" ] && doNetConf=y # digitalocean requires detailed network config to be generated +set -o pipefail +set -o nounset +set -o xtrace +set -o errexit apt update -apt install -y git tar wget curl whois jq +apt install -y git tar wget curl jq checkEnv prepareEnv makeSwap # smallest (512MB) droplet needs extra memory! -makeConf +setupConf infect removeSwap From b2edfdb78734104aee9b1034f0ccfd06957e0afc Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:07:28 +0400 Subject: [PATCH 012/124] route logs to another machine using netcat --- .drone.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 2eefbc7..16789f3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,14 +13,17 @@ steps: from_secret: CLOUDFLARE_TOKEN USER_PASS: from_secret: USER_PASS + DRONE_SOURCE_BRANCH: $DRONE_SOURCE_BRANCH commands: # Create infect user script and then export it to a remote machine on server creation. - echo '#!/bin/sh' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh + - echo 'echo Hello, NetCat' >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/hetzner-flakes/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - echo "Wait for 100 seconds." - - sleep 100 + - sleep 60 - false - name: dns From 3b800a7731c08448c3bb0087a485c18a3b0df729 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:27:02 +0400 Subject: [PATCH 013/124] trying to get any logs --- .drone.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 16789f3..353043f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -17,9 +17,11 @@ steps: commands: # Create infect user script and then export it to a remote machine on server creation. - - echo '#!/bin/sh' > infect.sh + - echo '#! /usr/bin/env bash' > infect.sh + - echo 'echo from machine | nc -N 128.140.94.125 8443' >> infect.sh + - echo 'echo sleep 1' >> infect.sh - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - - echo 'echo Hello, NetCat' >> infect.sh + - echo 'echo "Hello, NetCat"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/hetzner-flakes/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - echo "Wait for 100 seconds." From da56a4625bb5efad645e9035ac04fc50cfe8fc47 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:43:41 +0400 Subject: [PATCH 014/124] remove SUDO_USER variable --- .drone.yml | 6 ++---- nixos-infect | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index 353043f..0feeb4e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,11 +18,9 @@ steps: commands: # Create infect user script and then export it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo 'echo from machine | nc -N 128.140.94.125 8443' >> infect.sh - - echo 'echo sleep 1' >> infect.sh - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - - echo 'echo "Hello, NetCat"' >> infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/hetzner-flakes/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo 'echo "Hello, NetCat!"' >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - echo "Wait for 100 seconds." - sleep 60 diff --git a/nixos-infect b/nixos-infect index 1c2878f..d3399e2 100755 --- a/nixos-infect +++ b/nixos-infect @@ -16,7 +16,6 @@ : "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}" -: "${SUDO_USER:?SUDO_USER variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" readonly LOCAL_FLAKE_DIR="/etc/nixos" From 5131cfe35e6f4282027730a50a8ca74740fbd9bb Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:47:00 +0400 Subject: [PATCH 015/124] fix LADMIN => LUSER --- .drone.yml | 3 +-- nixos-infect | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0feeb4e..63143a8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,8 +22,7 @@ steps: - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - - echo "Wait for 100 seconds." - - sleep 60 + - sleep 140 - false - name: dns diff --git a/nixos-infect b/nixos-infect index d3399e2..bd295fa 100755 --- a/nixos-infect +++ b/nixos-infect @@ -39,7 +39,7 @@ genUserdata() { "domain": "DOMAIN", "hashedMasterPassword": "$HASHED_PASSWORD", "hostname": "HOSTNAME", - "username": "$LADMIN" + "username": "$LUSER" } EOF ) From e7512f3dc6f20caccc1f143117614a6533e6431a Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:49:17 +0400 Subject: [PATCH 016/124] NIX_VERSION=2.18.2 => NIX_VERSION=2.18.1 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 63143a8..5fbee84 100644 --- a/.drone.yml +++ b/.drone.yml @@ -20,7 +20,7 @@ steps: - echo '#! /usr/bin/env bash' > infect.sh - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.2 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - sleep 140 - false From db7dd2cff828c26ad5797e1b633d043dcba974e8 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:54:10 +0400 Subject: [PATCH 017/124] remove unneeded ubuntu repositories to speed up --- nixos-infect | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixos-infect b/nixos-infect index bd295fa..53d0e66 100755 --- a/nixos-infect +++ b/nixos-infect @@ -348,6 +348,9 @@ set -o nounset set -o xtrace set -o errexit +add-apt-repository --remove multiverse +add-apt-repository --remove universe +add-apt-repository --remove restricted apt update apt install -y git tar wget curl jq checkEnv From 5c9a185238069aa67d2d1d5d3277d9ba663d70d6 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 07:58:53 +0400 Subject: [PATCH 018/124] install multiuser Nix (--daemon) --- nixos-infect | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nixos-infect b/nixos-infect index 53d0e66..2a26b73 100755 --- a/nixos-infect +++ b/nixos-infect @@ -263,7 +263,7 @@ installNix() { fi echo "Running nix installer..." - if $SHELL "${tmpNixInstall}"; then + if $SHELL "${tmpNixInstall}" --daemon; then echo "Nix is installed" rm "${tmpNixInstall}" else @@ -348,9 +348,6 @@ set -o nounset set -o xtrace set -o errexit -add-apt-repository --remove multiverse -add-apt-repository --remove universe -add-apt-repository --remove restricted apt update apt install -y git tar wget curl jq checkEnv From 623856265c85480af3a72b8f1deac1ec61dd9e9e Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:08:06 +0400 Subject: [PATCH 019/124] name=ci-sibling => name=ci-sibling2 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5fbee84..3561d78 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling2 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - sleep 140 - false From 04260e08521805f01dc116b56f1dd9dc41f3ac65 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:15:13 +0400 Subject: [PATCH 020/124] nix flake lock --update-input sp-modules --- nixos-infect | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 2a26b73..f23665e 100755 --- a/nixos-infect +++ b/nixos-infect @@ -263,7 +263,8 @@ installNix() { fi echo "Running nix installer..." - if $SHELL "${tmpNixInstall}" --daemon; then + if $SHELL "${tmpNixInstall}" \ + --daemon --no-channel-add --daemon-user-count 4; then echo "Nix is installed" rm "${tmpNixInstall}" else @@ -295,6 +296,9 @@ infect() { # is it needed? # source ~/.nix-profile/etc/profile.d/nix.sh + cd ${LOCAL_FLAKE_DIR} + /root/.nix-profile/bin/nix flake lock --update-input sp-modules + echo "nix build the configuration flake..." if ! /root/.nix-profile/bin/nix build \ --extra-experimental-features "nix-command flakes" \ From 490cd38e4e98115e0584c1a1d44cdfb1ba495a69 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:17:21 +0400 Subject: [PATCH 021/124] name=ci-sibling2 => name=ci-sibling3 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3561d78..6000e47 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling2 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling3 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - sleep 140 - false From 9f23164bdafa63e4413d116c52d2c83388d17002 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:20:26 +0400 Subject: [PATCH 022/124] --extra-experimental-features "nix-command flakes" --- nixos-infect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index f23665e..c7e50db 100755 --- a/nixos-infect +++ b/nixos-infect @@ -297,7 +297,9 @@ infect() { # source ~/.nix-profile/etc/profile.d/nix.sh cd ${LOCAL_FLAKE_DIR} - /root/.nix-profile/bin/nix flake lock --update-input sp-modules + /root/.nix-profile/bin/nix flake lock \ + --extra-experimental-features "nix-command flakes" \ + --update-input sp-modules echo "nix build the configuration flake..." if ! /root/.nix-profile/bin/nix build \ From 3bf02d9f7289ca9721e5d5970eda3f95836c173b Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:21:07 +0400 Subject: [PATCH 023/124] name=ci-sibling3 => name=ci-sibling4 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 6000e47..ded7842 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling3 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling4 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - sleep 140 - false From 8f2d7c8cdb439a30cf2f2171babcba01be1044a2 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:31:30 +0400 Subject: [PATCH 024/124] name=ci-sibling4 => name=ci-sibling --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index ded7842..5fbee84 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling4 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - sleep 140 - false From 5c8f67f61bb7de70d4cd39a52931b1a0ea2b1dc8 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:35:23 +0400 Subject: [PATCH 025/124] without double quotes --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index c7e50db..067beb0 100755 --- a/nixos-infect +++ b/nixos-infect @@ -31,7 +31,7 @@ genUserdata() { { "dns": { "provider": "$DNS_PROVIDER_TYPE", - "useStagingACME": "$STAGING_ACME" + "useStagingACME": $STAGING_ACME }, "server": { "provider": "$PROVIDER" From 47f3ba8cee8a72b67ea28ed82e9103061ffe5b8c Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:41:14 +0400 Subject: [PATCH 026/124] increase wait time to 300 seconds --- .drone.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 5fbee84..84af78b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,8 +22,6 @@ steps: - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - - sleep 140 - - false - name: dns environment: @@ -77,8 +75,7 @@ steps: - name: sleep commands: - - echo "Wait for 100 seconds." - - sleep 100 + - sleep 300 - name: test environment: From 3fa57fc77f14d8533e75c6fadb63f83343f6cbb7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 08:58:34 +0400 Subject: [PATCH 027/124] rm -v -rf /etc/{dbus-1,terminfo,systemd} --- nixos-infect | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos-infect b/nixos-infect index 067beb0..1b3702e 100755 --- a/nixos-infect +++ b/nixos-infect @@ -311,6 +311,8 @@ infect() { exit 1 fi + rm -v -rf /etc/{dbus-1,terminfo,systemd} + echo "activate the built configuration..." if ! /nix/var/nix/profiles/system/activate; then echo "Failed!"; exit 1 From f3aca43b24b0064923c0164084512a7bdb0648e3 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 09:14:02 +0400 Subject: [PATCH 028/124] increase wait time to 360 seconds --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 84af78b..8aa5f20 100644 --- a/.drone.yml +++ b/.drone.yml @@ -75,7 +75,7 @@ steps: - name: sleep commands: - - sleep 300 + - sleep 360 - name: test environment: From 89b1f212ea6270d50165ab3e107eae5bfa1dba8d Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 17:28:19 +0400 Subject: [PATCH 029/124] do not activate or switch configuration (only boot) --- nixos-infect | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nixos-infect b/nixos-infect index 1b3702e..c72126c 100755 --- a/nixos-infect +++ b/nixos-infect @@ -311,18 +311,9 @@ infect() { exit 1 fi + # remove original OS stuff rm -v -rf /etc/{dbus-1,terminfo,systemd} - echo "activate the built configuration..." - if ! /nix/var/nix/profiles/system/activate; then - echo "Failed!"; exit 1 - fi - - echo "make configuration switch..." - if /nix/var/nix/profiles/system/bin/switch-to-configuration switch; then - echo "Failed!"; exit 1 - fi - # Remove nix installed by the "install" script. rm -fv /nix/var/nix/profiles/default* /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage From 4c0072fa7c3d29c0d1fe0a7072aa1513aced879d Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 17:36:06 +0400 Subject: [PATCH 030/124] do not install wget --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index c72126c..c2d0a91 100755 --- a/nixos-infect +++ b/nixos-infect @@ -348,7 +348,7 @@ set -o xtrace set -o errexit apt update -apt install -y git tar wget curl jq +apt install -y git tar curl jq checkEnv prepareEnv makeSwap # smallest (512MB) droplet needs extra memory! From c51aece2b9e0fbd9d81f966b6facb8f40b7e0e3c Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 17:43:36 +0400 Subject: [PATCH 031/124] fix switch-to-configuration boot exit code check --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index c2d0a91..273f77e 100755 --- a/nixos-infect +++ b/nixos-infect @@ -337,7 +337,7 @@ infect() { fi echo "make configuration boot by default..." - if /nix/var/nix/profiles/system/bin/switch-to-configuration boot; then + if ! /nix/var/nix/profiles/system/bin/switch-to-configuration boot; then echo "Failed!"; exit 1 fi } From 4d62fb57b8f6772fb65dca5eaa77fc872a75add5 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 18:21:34 +0400 Subject: [PATCH 032/124] fix NO_REBOOT check in the presense of the "nounset" shell option --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 273f77e..f4eabd9 100755 --- a/nixos-infect +++ b/nixos-infect @@ -356,6 +356,6 @@ setupConf infect removeSwap -if [[ -z "$NO_REBOOT" ]]; then +if [[ -z "${NO_REBOOT+x}" ]]; then reboot fi From 5da925ea6672484db7db2bc2e41fb5a8d3ee9734 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 18:49:05 +0400 Subject: [PATCH 033/124] drone: delete all 4 used machines --- .drone.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8aa5f20..0419e08 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: from_secret: ZONE_ID commands: - # Read all available machines and export IP of 'ci-sibling' + # Read all available machines and export IP of 'ci-sibling' # Create DNS entires for main domain and subdomains - > curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json @@ -45,7 +45,7 @@ steps: - > curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" - -H "Content-Type: application/json" + -H "Content-Type: application/json" - > curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" @@ -121,6 +121,18 @@ steps: export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - > curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" + - > + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + - > + curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" + - > + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling3" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + - > + curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" + - > + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling4" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + - > + curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - > json=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json") - > From d73fdda3ade57687d1fa91afd50af2ee142da448 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 18:59:42 +0400 Subject: [PATCH 034/124] Revert "drone: delete all 4 used machines" This reverts commit 5da925ea6672484db7db2bc2e41fb5a8d3ee9734. --- .drone.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0419e08..8aa5f20 100644 --- a/.drone.yml +++ b/.drone.yml @@ -35,7 +35,7 @@ steps: from_secret: ZONE_ID commands: - # Read all available machines and export IP of 'ci-sibling' + # Read all available machines and export IP of 'ci-sibling' # Create DNS entires for main domain and subdomains - > curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json @@ -45,7 +45,7 @@ steps: - > curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" - -H "Content-Type: application/json" + -H "Content-Type: application/json" - > curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" @@ -121,18 +121,6 @@ steps: export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - > curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - - > - export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - - > - curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - - > - export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling3" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - - > - curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - - > - export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling4" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - - > - curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - > json=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json") - > From 97b9a8421bd314a46f34090382d3650b540df061 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 18:32:35 +0400 Subject: [PATCH 035/124] ci-sibling => ci-sibling2 --- .drone.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8aa5f20..74401aa 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling2 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns environment: @@ -35,17 +35,17 @@ steps: from_secret: ZONE_ID commands: - # Read all available machines and export IP of 'ci-sibling' + # Read all available machines and export IP of 'ci-sibling2' # Create DNS entires for main domain and subdomains - > curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json - > - export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' ) + export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' ) - echo $machineip - > curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" - -H "Content-Type: application/json" + -H "Content-Type: application/json" - > curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" @@ -114,11 +114,11 @@ steps: ZONE_ID: from_secret: ZONE_ID commands: - # Delete our machine called 'ci-sibling' and all the found DNS records on it. + # Delete our machine called 'ci-sibling2' and all the found DNS records on it. - > curl -s "https://api.hetzner.cloud/v1/servers" -H "Authorization: Bearer $PASSWORD" > .hetzner_servers.json - > - export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - > curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - > From eeb9b257bf04b695896138a3ed34f830f49801e7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 19:04:58 +0400 Subject: [PATCH 036/124] print hardware-configuration.nix in the end --- .drone.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.drone.yml b/.drone.yml index 74401aa..612c4f8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,6 +21,7 @@ steps: - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo 'cat /etc/nixos/hardware-configuration.nix' >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling2 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns From fdfd074e2769c09b664f7d2aab67ff1328e781e6 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 19:16:32 +0400 Subject: [PATCH 037/124] drone: ci-sibling2 => ci-sibling --- .drone.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index 612c4f8..a721b1d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,7 +22,7 @@ steps: - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - echo 'cat /etc/nixos/hardware-configuration.nix' >> infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling2 server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns environment: @@ -36,12 +36,12 @@ steps: from_secret: ZONE_ID commands: - # Read all available machines and export IP of 'ci-sibling2' + # Read all available machines and export IP of 'ci-sibling' # Create DNS entires for main domain and subdomains - > curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json - > - export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' ) + export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' ) - echo $machineip - > curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN" @@ -115,11 +115,11 @@ steps: ZONE_ID: from_secret: ZONE_ID commands: - # Delete our machine called 'ci-sibling2' and all the found DNS records on it. + # Delete our machine called 'ci-sibling' and all the found DNS records on it. - > curl -s "https://api.hetzner.cloud/v1/servers" -H "Authorization: Bearer $PASSWORD" > .hetzner_servers.json - > - export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling2" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') + export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g') - > curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD" - > From daf4390cc9cccc1d76bedc78eab3b8e1cde091ae Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 18 Nov 2023 19:28:33 +0400 Subject: [PATCH 038/124] do not cd into ${LOCAL_FLAKE_DIR} --- nixos-infect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index f4eabd9..cf277ed 100755 --- a/nixos-infect +++ b/nixos-infect @@ -296,8 +296,7 @@ infect() { # is it needed? # source ~/.nix-profile/etc/profile.d/nix.sh - cd ${LOCAL_FLAKE_DIR} - /root/.nix-profile/bin/nix flake lock \ + /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ --update-input sp-modules From c58597c4d763731df163795dbe198d36bd7ac922 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 20 Nov 2023 18:59:00 +0400 Subject: [PATCH 039/124] comment about flake lock and subflake --- nixos-infect | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos-infect b/nixos-infect index cf277ed..34ce6e9 100755 --- a/nixos-infect +++ b/nixos-infect @@ -296,6 +296,7 @@ infect() { # is it needed? # source ~/.nix-profile/etc/profile.d/nix.sh + # this is needed solely for accpeting the sp-module subflake /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ --update-input sp-modules From 31dfa8eb4af5f333bb73bf1967e06cc68d827283 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 20 Nov 2023 19:02:32 +0400 Subject: [PATCH 040/124] add users.users.root.hashedPassword for testing purposes --- nixos-infect | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos-infect b/nixos-infect index 34ce6e9..eaa35b7 100755 --- a/nixos-infect +++ b/nixos-infect @@ -101,6 +101,8 @@ $bootcfg fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; }; } EOF +# FIXME remove this! +echo '// { users.users.root.hashedPassword = "$6$I8xOgBRfitytj331$WZzyJbABCY8LZ4CqpUzNU2dXK8DP8rdVXYms60c0ysINREEFB49KYL23E.twuQ..beV2yb6VAwZkxDymCFOSO/"; }' } setupConf() { From ce8d1f84040fc094af8851372b3aceeacbf442ca Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 20 Nov 2023 19:02:54 +0400 Subject: [PATCH 041/124] drone: DRONE_SOURCE_BRANCH => INFECT_SOURCE_BRANCH --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index a721b1d..3226143 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,14 +13,14 @@ steps: from_secret: CLOUDFLARE_TOKEN USER_PASS: from_secret: USER_PASS - DRONE_SOURCE_BRANCH: $DRONE_SOURCE_BRANCH + INFECT_SOURCE_BRANCH: $DRONE_SOURCE_BRANCH commands: # Create infect user script and then export it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - echo 'echo "Hello, NetCat!"' >> infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$DRONE_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - echo 'cat /etc/nixos/hardware-configuration.nix' >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 From 73b5863968c86d56da1d4cbfdc28083330d74c77 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 00:25:34 +0400 Subject: [PATCH 042/124] fix isEFI variable assignment --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index eaa35b7..be784e4 100755 --- a/nixos-infect +++ b/nixos-infect @@ -157,7 +157,7 @@ findESP() { prepareEnv() { isEFI=0 - [ -d /sys/firmware/efi ] || isEFI=1 + [ -d /sys/firmware/efi ] && isEFI=1 if ((isEFI)); then ESP="$(findESP)" From bc0f59695c8b3ff0bcd813e499464bb050d24b8f Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 00:57:54 +0400 Subject: [PATCH 043/124] exclude dot files when extracting top-level flake --- nixos-infect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index be784e4..9eb8587 100755 --- a/nixos-infect +++ b/nixos-infect @@ -107,7 +107,8 @@ echo '// { users.users.root.hashedPassword = "$6$I8xOgBRfitytj331$WZzyJbABCY8LZ4 setupConf() { mkdir -p ${LOCAL_FLAKE_DIR} - if ! curl "${CONFIG_URL}" | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 + if ! curl "${CONFIG_URL}" \ + | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 --exclude=".*" then echo "Error downloading/extracting top level flake configuration!" exit 1 From 5169969e039dd5f2ce3d2e1d9589b0f0b6caf41a Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 01:18:51 +0400 Subject: [PATCH 044/124] fix passing HOSTNAME and DOMAIN variables to userdata.json --- nixos-infect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index 9eb8587..097ce7d 100755 --- a/nixos-infect +++ b/nixos-infect @@ -36,9 +36,9 @@ genUserdata() { "server": { "provider": "$PROVIDER" }, - "domain": "DOMAIN", + "domain": "$DOMAIN", "hashedMasterPassword": "$HASHED_PASSWORD", - "hostname": "HOSTNAME", + "hostname": "$HOSTNAME", "username": "$LUSER" } EOF From 7e79a267151dce11a429193ef0d3467abe59254c Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 02:52:06 +0400 Subject: [PATCH 045/124] do not keep root/.nix-defexpr/channels after reboot --- nixos-infect | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/nixos-infect b/nixos-infect index 097ce7d..a8f8e6d 100755 --- a/nixos-infect +++ b/nixos-infect @@ -277,19 +277,7 @@ installNix() { } infect() { - # Add nix build users - # FIXME run only if necessary, rather than defaulting true - # groupadd nixbld -g 30000 || true - # for i in {1..10}; do - # useradd -c "Nix build user $i" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" "nixbld$i" || true - # done - # TODO use addgroup and adduser as fallbacks - #addgroup nixbld -g 30000 || true - #for i in {1..10}; do adduser -DH -G nixbld nixbld$i || true; done - - # install Nix in the current system - # (this should create system nixbld users too) - # curl -L "https://releases.nixos.org/nix/nix-$NIX_VERSION/install" | $SHELL + # install multiuser (system-wide with nix-daemon) Nix in the current system if ! installNix; then echo "Nix installation failed!" exit 1 @@ -326,9 +314,8 @@ infect() { # Stage the Nix coup d'état touch /etc/NIXOS - echo etc/nixos > /etc/NIXOS_LUSTRATE - echo etc/resolv.conf >> /etc/NIXOS_LUSTRATE - echo root/.nix-defexpr/channels >> /etc/NIXOS_LUSTRATE + echo etc/nixos > /etc/NIXOS_LUSTRATE + echo etc/resolv.conf >> /etc/NIXOS_LUSTRATE rm -rf /boot.bak ((isEFI)) && umount "$ESP" From 6645c8e61bfdfb66e07303ec5dae64feb28c15a7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 02:52:23 +0400 Subject: [PATCH 046/124] Revert "add users.users.root.hashedPassword for testing purposes" This reverts commit 31dfa8eb4af5f333bb73bf1967e06cc68d827283. --- nixos-infect | 2 -- 1 file changed, 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index a8f8e6d..e99b4f3 100755 --- a/nixos-infect +++ b/nixos-infect @@ -101,8 +101,6 @@ $bootcfg fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; }; } EOF -# FIXME remove this! -echo '// { users.users.root.hashedPassword = "$6$I8xOgBRfitytj331$WZzyJbABCY8LZ4CqpUzNU2dXK8DP8rdVXYms60c0ysINREEFB49KYL23E.twuQ..beV2yb6VAwZkxDymCFOSO/"; }' } setupConf() { From 401f6e07897c741025deaae3ee218f2e753c4bc2 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 06:31:02 +0400 Subject: [PATCH 047/124] do not remove original OS stuff (it will be moved to /old-root anyway) --- nixos-infect | 7 ------- 1 file changed, 7 deletions(-) diff --git a/nixos-infect b/nixos-infect index e99b4f3..9615b26 100755 --- a/nixos-infect +++ b/nixos-infect @@ -281,10 +281,6 @@ infect() { exit 1 fi - # shellcheck disable=SC1090 - # is it needed? - # source ~/.nix-profile/etc/profile.d/nix.sh - # this is needed solely for accpeting the sp-module subflake /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ @@ -300,9 +296,6 @@ infect() { exit 1 fi - # remove original OS stuff - rm -v -rf /etc/{dbus-1,terminfo,systemd} - # Remove nix installed by the "install" script. rm -fv /nix/var/nix/profiles/default* /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage From cae1b6468985f1ac6cb3e736a07f22c48471eb94 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 21 Nov 2023 15:28:21 +0200 Subject: [PATCH 048/124] fix infect: lustrate /etc/selfprivacy/secrets.json --- nixos-infect | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index 9615b26..cc41a03 100755 --- a/nixos-infect +++ b/nixos-infect @@ -305,8 +305,9 @@ infect() { # Stage the Nix coup d'état touch /etc/NIXOS - echo etc/nixos > /etc/NIXOS_LUSTRATE - echo etc/resolv.conf >> /etc/NIXOS_LUSTRATE + echo etc/nixos > /etc/NIXOS_LUSTRATE + echo etc/resolv.conf >> /etc/NIXOS_LUSTRATE + echo ${SECRETS_FILEPATH} >> /etc/NIXOS_LUSTRATE rm -rf /boot.bak ((isEFI)) && umount "$ESP" From 8462fff065ce4ef36c215ea846de6689c70c97ef Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 21 Nov 2023 07:04:58 +0400 Subject: [PATCH 049/124] drone: sleep duration 360 => 600 seconds --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 3226143..d8eb516 100644 --- a/.drone.yml +++ b/.drone.yml @@ -76,7 +76,7 @@ steps: - name: sleep commands: - - sleep 360 + - sleep 600 - name: test environment: From 4cf9f98105283cb6a3b252c1fab3a6cd63e0d1f7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 22 Nov 2023 01:49:05 +0400 Subject: [PATCH 050/124] remove installed Nix after switch-to-configuration boot --- nixos-infect | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nixos-infect b/nixos-infect index cc41a03..a5c2a89 100755 --- a/nixos-infect +++ b/nixos-infect @@ -296,10 +296,6 @@ infect() { exit 1 fi - # Remove nix installed by the "install" script. - rm -fv /nix/var/nix/profiles/default* - /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage - # Reify resolv.conf (???) [[ -L /etc/resolv.conf ]] && mv -v /etc/resolv.conf /etc/resolv.conf.lnk && cat /etc/resolv.conf.lnk > /etc/resolv.conf @@ -322,6 +318,10 @@ infect() { if ! /nix/var/nix/profiles/system/bin/switch-to-configuration boot; then echo "Failed!"; exit 1 fi + + # Remove nix installed by the "install" script. + rm -fv /nix/var/nix/profiles/default* + /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage } set -o pipefail From 5fc8a090f705cb7d1f41584e65ad41b187cd3c13 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 22 Nov 2023 22:26:58 +0400 Subject: [PATCH 051/124] add github issue link to the comment about subflake --- nixos-infect | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos-infect b/nixos-infect index a5c2a89..b111950 100755 --- a/nixos-infect +++ b/nixos-infect @@ -282,6 +282,7 @@ infect() { fi # this is needed solely for accpeting the sp-module subflake + # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ --update-input sp-modules From 347ee5ca121388d7232a63f3f3ecf09d99e3e1cd Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 22 Nov 2023 23:50:09 +0400 Subject: [PATCH 052/124] fix typo in generated secrets.json --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index b111950..bee7908 100755 --- a/nixos-infect +++ b/nixos-infect @@ -65,7 +65,7 @@ genSecrets() { "modules": { "nextcloud": { "adminPassword": $ESCAPED_PASSWORD, - "databasePassword": $ESCAPED_PASSWORD, + "databasePassword": $ESCAPED_PASSWORD } }, "resticPassword": $ESCAPED_PASSWORD From 212f2806f0b9500e2e545587488b883ada192bae Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 23 Nov 2023 03:18:44 +0400 Subject: [PATCH 053/124] sleep 1000 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d8eb516..98e8e77 100644 --- a/.drone.yml +++ b/.drone.yml @@ -76,7 +76,7 @@ steps: - name: sleep commands: - - sleep 600 + - sleep 1000 - name: test environment: From 670985f3d59868b3a203c83ef348693f162c08c1 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 23 Nov 2023 21:22:41 +0400 Subject: [PATCH 054/124] do not send nixos-infect logs over netcat --- .drone.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 98e8e77..f1e00c1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,10 +18,7 @@ steps: commands: # Create infect user script and then export it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo 'exec &> >(tee >(nc -N 128.140.94.125 8443))' >> infect.sh - - echo 'echo "Hello, NetCat!"' >> infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - - echo 'cat /etc/nixos/hardware-configuration.nix' >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns From aee0eab1784cc7d1e327e0a0b4479b5a039ba184 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 04:14:30 +0400 Subject: [PATCH 055/124] sleep 999 (dummy commit for CI) --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index f1e00c1..5d05010 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 1000 + - sleep 999 - name: test environment: From 6c0486034d4e738d7cf3371b03284374e113b50e Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 06:21:38 +0400 Subject: [PATCH 056/124] drone: fix INFECT_SOURCE_BRANCH variable --- .drone.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 5d05010..cf57054 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,10 +13,10 @@ steps: from_secret: CLOUDFLARE_TOKEN USER_PASS: from_secret: USER_PASS - INFECT_SOURCE_BRANCH: $DRONE_SOURCE_BRANCH + INFECT_SOURCE_BRANCH: ${DRONE_SOURCE_BRANCH} commands: - # Create infect user script and then export it to a remote machine on server creation. + # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 999 + - sleep 300 - name: test environment: From 00cf7c40a1e42d1ead3f031bad3fb076ee5d8fe9 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 06:34:15 +0400 Subject: [PATCH 057/124] drone: sleep 360 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index cf57054..61ca885 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 300 + - sleep 360 - name: test environment: From 66c0e152938cd6f5f2af32238ed525c86086f067 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 06:57:20 +0400 Subject: [PATCH 058/124] drone: sleep 1000 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 61ca885..7bd1c02 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 360 + - sleep 1000 - name: test environment: From fd924f354f9ecd56537a626010d07b814a3b895f Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 13:53:20 +0400 Subject: [PATCH 059/124] Revert "drone: sleep 1000" This reverts commit 66c0e152938cd6f5f2af32238ed525c86086f067. --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 7bd1c02..61ca885 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 1000 + - sleep 360 - name: test environment: From 00f9141343434919f5f9d0ef4f7edad470a62ef5 Mon Sep 17 00:00:00 2001 From: Inex Code Date: Fri, 24 Nov 2023 13:20:45 +0300 Subject: [PATCH 060/124] ci: Hide email password --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 61ca885..97fa309 100644 --- a/.drone.yml +++ b/.drone.yml @@ -89,7 +89,7 @@ steps: - > http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - > - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS + swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - > http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - > From 40bade435bfe2dd06f28573d6bd95566414c7ca6 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 16:35:14 +0400 Subject: [PATCH 061/124] drone: change nixos template URL --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 97fa309..00ad5b6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,7 +18,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/alexoundos/selfprivacy-nixos-top-level/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns From 79bb94d7e4d73e98ba35ac209cac59a8bd980905 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 24 Nov 2023 17:04:38 +0400 Subject: [PATCH 062/124] drone: sleep 420 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 00ad5b6..2809d4c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 360 + - sleep 420 - name: test environment: From ac5a9c35174bea24b5c231f1e225cb90fdade763 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 25 Nov 2023 23:05:34 +0400 Subject: [PATCH 063/124] drone: sleep 600 --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2809d4c..19d8be6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -73,7 +73,7 @@ steps: - name: sleep commands: - - sleep 420 + - sleep 600 - name: test environment: From ccf32a3172a8baf33a31785365d41a8cfb5ea704 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 29 Nov 2023 08:53:23 +0400 Subject: [PATCH 064/124] drone: pass USER_PASS without base64 --- .drone.yml | 2 +- nixos-infect | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.drone.yml b/.drone.yml index 19d8be6..5147cf3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -18,7 +18,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd ENCODED_PASSWORD=\"$(printf $USER_PASS | base64)\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=$USER_PASS API_TOKEN=$USER_PASS HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns diff --git a/nixos-infect b/nixos-infect index bee7908..06bf51a 100755 --- a/nixos-infect +++ b/nixos-infect @@ -13,7 +13,7 @@ : "${STAGING_ACME:?STAGING_ACME variable is not set}" : "${CF_TOKEN:?CF_TOKEN variable is not set}" : "${DB_PASSWORD:?DB_PASSWORD variable is not set}" -: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" +: "${USER_PASS:?USER_PASS variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" @@ -23,9 +23,8 @@ readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json" # Merge original userdata.json with deployment specific fields and print result. genUserdata() { - local password HASHED_PASSWORD userdata_infect - password=$(printf "%s" "$ENCODED_PASSWORD" | base64 --decode) - HASHED_PASSWORD=$(mkpasswd -m sha-512 "$password") + local HASHED_PASSWORD userdata_infect + HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")" userdata_infect=$(cat << EOF { @@ -49,9 +48,6 @@ EOF } genSecrets() { - local ESCAPED_PASSWORD - ESCAPED_PASSWORD=$(printf "%s" "$ENCODED_PASSWORD" | base64 --decode | jq -Rs .) - cat << EOF { "api": { @@ -64,11 +60,11 @@ genSecrets() { }, "modules": { "nextcloud": { - "adminPassword": $ESCAPED_PASSWORD, - "databasePassword": $ESCAPED_PASSWORD + "adminPassword": "$USER_PASS", + "databasePassword": "$USER_PASS" } }, - "resticPassword": $ESCAPED_PASSWORD + "resticPassword": "$USER_PASS" } EOF } From 55149c1af0e28ab6147e39957edc3cc9eaaf66ed Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sun, 3 Dec 2023 07:08:59 +0400 Subject: [PATCH 065/124] infect: fix typo in comment --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 06bf51a..05991ef 100755 --- a/nixos-infect +++ b/nixos-infect @@ -277,7 +277,7 @@ infect() { exit 1 fi - # this is needed solely for accpeting the sp-module subflake + # this is needed solely for accepting the sp-module subflake # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ From 9245d6584da1c352556453c69fb4a583c4137771 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 4 Dec 2023 17:09:11 +0400 Subject: [PATCH 066/124] drone: clone disable --- .drone.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.drone.yml b/.drone.yml index 5147cf3..08dcc0a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,6 +2,9 @@ kind: pipeline type: exec name: default +clone: + disable: true + steps: - name: deploy environment: From 8de4c0908fdfe6b7762914e1f3a1227e5a2b2c59 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 5 Dec 2023 05:28:15 +0400 Subject: [PATCH 067/124] pass SSH_AUTHORIZED_KEY and merge with userdata.json --- .drone.yml | 2 +- nixos-infect | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 08dcc0a..dbcf7e0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns diff --git a/nixos-infect b/nixos-infect index 05991ef..6ec6d35 100755 --- a/nixos-infect +++ b/nixos-infect @@ -17,10 +17,17 @@ : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" +: "${SSH_AUTHORIZED_KEY:=}" readonly LOCAL_FLAKE_DIR="/etc/nixos" readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json" +genOptionalSsh() { + [ -n "${SSH_AUTHORIZED_KEY}" ] && cat << EOF +"ssh": { "rootKeys": [ "${SSH_AUTHORIZED_KEY}" ] }, +EOF +} + # Merge original userdata.json with deployment specific fields and print result. genUserdata() { local HASHED_PASSWORD userdata_infect @@ -28,6 +35,7 @@ genUserdata() { userdata_infect=$(cat << EOF { + $(genOptionalSsh) "dns": { "provider": "$DNS_PROVIDER_TYPE", "useStagingACME": $STAGING_ACME From ba181fb1626818f771faba57b67a2d73882d3d54 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 5 Dec 2023 07:30:19 +0400 Subject: [PATCH 068/124] generate deployment.nix with system.stateVersion --- nixos-infect | 57 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/nixos-infect b/nixos-infect index 6ec6d35..0659368 100755 --- a/nixos-infect +++ b/nixos-infect @@ -96,6 +96,17 @@ EOF ) fi + local NIX="/root/.nix-profile/bin/nix" + local NIX_OPTS=(--extra-experimental-features nix-command) + local currentSystem + if ! currentSystem="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr builtins.currentSystem)" + then + echo "cannot determine Nix currentSystem identifier" + return 1 + fi + + # TODO try nixos-generate-config first, resorting to the way below if failed + cat << EOF { modulesPath, ... }: { @@ -103,6 +114,28 @@ EOF $bootcfg boot.initrd.kernelModules = [ "nvme" ]; fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; }; + nixpkgs.hostPlatform = "${currentSystem}"; +} +EOF +} + +genDeploymentConfiguration() { + local NIX="/root/.nix-profile/bin/nix" + local NIX_OPTS=(--extra-experimental-features "nix-command flakes") + local release + + if ! release="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr "(builtins.getFlake (builtins.toString ${LOCAL_FLAKE_DIR})).inputs.selfprivacy-nixos-config.inputs.nixpkgs.lib.trivial.release")" + then + echo "cannot determine NixOS release version" + return 1 + fi + + cat << EOF +{ + # The content below is static and belongs to this deployment only! + # Do not copy this configuration file to another NixOS installation! + + system.stateVersion = "$release"; } EOF } @@ -117,7 +150,18 @@ setupConf() { fi # generate and write hardware-configuration.nix - genHardwareConfiguration > ${LOCAL_FLAKE_DIR}/hardware-configuration.nix + if ! genHardwareConfiguration > ${LOCAL_FLAKE_DIR}/hardware-configuration.nix + then + echo "error generating ${LOCAL_FLAKE_DIR}/hardware-configuration.nix" + exit 1 + fi + + # generate and write deployment.nix + if ! genDeploymentConfiguration > ${LOCAL_FLAKE_DIR}/deployment.nix + then + echo "error generating ${LOCAL_FLAKE_DIR}/deployment.nix" + exit 1 + fi # generate infected userdata based on original local userdataInfected @@ -237,7 +281,10 @@ checkEnv() { } # Download and execute the nix installer script. + installNix() { + # install multiuser (system-wide with nix-daemon) Nix in the current system + local nixReleaseBase='https://releases.nixos.org' local installURL="${nixReleaseBase}/nix/nix-${NIX_VERSION}/install" local shaURL="${installURL}.sha256" @@ -279,12 +326,6 @@ installNix() { } infect() { - # install multiuser (system-wide with nix-daemon) Nix in the current system - if ! installNix; then - echo "Nix installation failed!" - exit 1 - fi - # this is needed solely for accepting the sp-module subflake # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ @@ -336,9 +377,11 @@ set -o errexit apt update apt install -y git tar curl jq + checkEnv prepareEnv makeSwap # smallest (512MB) droplet needs extra memory! +installNix setupConf infect removeSwap From 158c711f66be74e59166696dc33389d468e48b05 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 11 Dec 2023 13:40:42 +0400 Subject: [PATCH 069/124] fix hashedMasterPassword: install missing `mkpasswd` via `whois` package --- nixos-infect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index 0659368..93489c4 100755 --- a/nixos-infect +++ b/nixos-infect @@ -278,6 +278,7 @@ checkEnv() { req xzcat || { echo "ERROR: Missing xzcat"; return 1; } req awk || { echo "ERROR: Missing awk"; return 1; } req cut || req df || { echo "ERROR: Missing coreutils (cut, df)"; return 1; } + req mkpasswd || { echo "ERROR: Missing mkpasswd"; return 1; } } # Download and execute the nix installer script. @@ -374,9 +375,10 @@ set -o pipefail set -o nounset set -o xtrace set -o errexit +shopt -s inherit_errexit apt update -apt install -y git tar curl jq +apt install -y git tar curl whois jq checkEnv prepareEnv From 528d5de6dd0acd02ec1474e8878fa3e4910ec533 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 11 Dec 2023 15:55:46 +0400 Subject: [PATCH 070/124] networking configuration for DigitalOcean --- nixos-infect | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/nixos-infect b/nixos-infect index 93489c4..9448ed0 100755 --- a/nixos-infect +++ b/nixos-infect @@ -19,8 +19,10 @@ : "${CONFIG_URL:?CONFIG_URL variable is not set}" : "${SSH_AUTHORIZED_KEY:=}" +readonly NL=$'\n' readonly LOCAL_FLAKE_DIR="/etc/nixos" readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json" +DoNetConf= genOptionalSsh() { [ -n "${SSH_AUTHORIZED_KEY}" ] && cat << EOF @@ -131,11 +133,12 @@ genDeploymentConfiguration() { fi cat << EOF -{ +${DoNetConf:+"{ lib, ... }: "}{ # The content below is static and belongs to this deployment only! # Do not copy this configuration file to another NixOS installation! - system.stateVersion = "$release"; + system.stateVersion = "$release";` +`$(if [ "$DoNetConf" == "y" ]; then echo -e "$NL"; genNetworkingConf; fi) } EOF } @@ -174,6 +177,85 @@ setupConf() { install -m0600 <(printf "%s" "$secrets") -DT ${SECRETS_FILEPATH} } +# shellcheck disable=SC2207 +genNetworkingConf() { + # XXX It'd be better if we used procfs for all this... + + local IFS=$'\n' + local eth0_name eth0_ip4s eth0_ip6s gateway gateway6 ether0 eth1_name + local interfaces1 extraRules1 predictable_inames + + eth0_name="$(ip address show | grep '^2:' | awk -F': ' '{print $2}')" + eth0_ip4s=($(ip address show dev "$eth0_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) + eth0_ip6s=($(ip address show dev "$eth0_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) || true + gateway="$(ip route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9.]+).*|\1|')" + gateway6="$(ip -6 route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|')" || true + ether0="$(ip address show dev "$eth0_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')" + + eth1_name="$(ip address show | grep '^3:' | awk -F': ' '{print $2}')" || true + if [ -n "$eth1_name" ]; then + local eth1_ip4s eth1_ip6s ether1 + eth1_ip4s="$(ip address show dev "$eth1_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')" + eth1_ip6s="$(ip address show dev "$eth1_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')" || true + ether1="$(ip address show dev "$eth1_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')" + interfaces1=$(cat << EOF + $eth1_name = { + ipv4.addresses = [$(for a in "${eth1_ip4s[@]}"; do echo -n " + $a"; done) + ]; + ipv6.addresses = [$(for a in "${eth1_ip6s[@]}"; do echo -n " + $a"; done) + ]; + }; +EOF +) + extraRules1="ATTR{address}==\"${ether1}\", NAME=\"${eth1_name}\"" + else + interfaces1="" + extraRules1="" + fi + + if [[ "$eth0_name" = eth* ]]; then + predictable_inames="usePredictableInterfaceNames = lib.mkForce false;" + else + predictable_inames="usePredictableInterfaceNames = lib.mkForce true;" + fi + + local defaultGateway6=${gateway6:+defaultGateway6 = \{ address = "${gateway6}"; interface = "${eth0_name}"; \};} + local ipv6routes=${gateway6:+ipv6.routes = \[ \{ address = "${gateway6}"; prefixLength = 128; \} \];} + cat << EOF + # Networking configuration was populated by nixos-infect with the networking + # details gathered from the running system. + networking = { + defaultGateway = "${gateway}";` + `${defaultGateway6:+ + defaultGateway6} + dhcpcd.enable = false; + $predictable_inames + interfaces = { + $eth0_name = { + ipv4.addresses = [$(for a in "${eth0_ip4s[@]}"; do echo -n " + $a"; done) + ]; + ipv6.addresses = [$(for a in "${eth0_ip6s[@]}"; do echo -n " + $a"; done) + ]; + ipv4.routes = [ { address = "${gateway}"; prefixLength = 32; } ];` + `${ipv6routes:+ + $ipv6routes} + };` +`${interfaces1:+ +$interfaces1} + }; + }; + services.udev.extraRules = '' + ATTR{address}=="${ether0}", NAME="${eth0_name}"` + `${extraRules1:+ + $extraRules1} + ''; +EOF +} + makeSwap() { # TODO check currently available swapspace first swapFile=$(mktemp /tmp/nixos-infect.XXXXX.swp) @@ -373,10 +455,15 @@ infect() { set -o pipefail set -o nounset -set -o xtrace set -o errexit +set -o xtrace shopt -s inherit_errexit +genNetworkingConf + +# digitalocean requires detailed network config to be generated +[ "$PROVIDER" == "digitalocean" ] && DoNetConf="y" + apt update apt install -y git tar curl whois jq From fa062875e76106e892c4bcc40d9a716379af37bf Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 11 Dec 2023 21:11:14 +0400 Subject: [PATCH 071/124] add questionable availableKernelModules for DigitalOcean deployment --- nixos-infect | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nixos-infect b/nixos-infect index 9448ed0..87c165b 100755 --- a/nixos-infect +++ b/nixos-infect @@ -107,7 +107,17 @@ EOF return 1 fi + # FIXME it's questionable whether these modules are needed at all... + declare -a availableKernelModules=() + + [ "$PROVIDER" == "digitalocean" ] \ + && availableKernelModules+=('"ata_piix"' '"uhci_hcd"' '"xen_blkfront"') + + [ "$(uname -m)" == "x86_64" ] \ + && availableKernelModules+=('"vmw_pvscsi"') + # TODO try nixos-generate-config first, resorting to the way below if failed + # FIXME "nvme" is not needed for hetzner? cat << EOF { modulesPath, ... }: @@ -115,6 +125,7 @@ EOF imports = [ (modulesPath + "/profiles/qemu-guest.nix") ]; $bootcfg boot.initrd.kernelModules = [ "nvme" ]; + boot.initrd.availableKernelModules = [ ${availableKernelModules[@]} ]; fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; }; nixpkgs.hostPlatform = "${currentSystem}"; } From b49b801baabcf554c7381d9a2c3dfc0fc68701b7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 05:02:05 +0400 Subject: [PATCH 072/124] add test: query-all-services state via GraphQL API --- .drone.yml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.drone.yml b/.drone.yml index dbcf7e0..f86ca1e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -76,7 +76,7 @@ steps: - name: sleep commands: - - sleep 600 + - sleep 420 - name: test environment: @@ -87,22 +87,22 @@ steps: TEST_EMAIL_PASS: from_secret: TEST_EMAIL_PASS commands: - - > - dig api.$DOMAIN - - > - http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - - > - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - > - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - - > - echo "File to test nextcloud upload" > file - - > - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - - > - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - #- > - #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics + - dig api.$DOMAIN + - http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp + - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - echo "File to test nextcloud upload" > file + - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS + - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-all-services + #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics + +# - name: sleep-after-failure +# commands: +# - sleep 240 +# when: +# status: +# - failure - name: teardown environment: From 1008f2aa5166f93be4ad95e896786bc222cadb1b Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 09:08:04 +0400 Subject: [PATCH 073/124] pass API_TOKEN to query-all-services test --- .drone.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.drone.yml b/.drone.yml index f86ca1e..0e68382 100644 --- a/.drone.yml +++ b/.drone.yml @@ -86,6 +86,8 @@ steps: from_secret: USER_PASS TEST_EMAIL_PASS: from_secret: TEST_EMAIL_PASS + API_TOKEN: + from_secret: USER_PASS commands: - dig api.$DOMAIN - http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" From c130acf95a13574ca6651244924291d6b6cf33df Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 12:18:46 +0400 Subject: [PATCH 074/124] enable services via API, wait until active or timeout --- .drone.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 0e68382..34aabbc 100644 --- a/.drone.yml +++ b/.drone.yml @@ -96,7 +96,14 @@ steps: - echo "File to test nextcloud upload" > file - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-all-services + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-minimum-services + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service bitwarden + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service gitea + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service jitsi # - FIXME it should be jitsi-meet + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service ocserv + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service pleroma + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#nixos-rebuild Rebuild + - counter=1; while true; do sleep 4; nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-all-services && break; [ "$counter" -le 30 ]; ((counter++)); done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics # - name: sleep-after-failure From 94421ee3d5056b5e84b626d615a75296c6dd46c6 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 14:32:55 +0400 Subject: [PATCH 075/124] jitsi => jitsi-meet --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 34aabbc..aaeaed9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -99,7 +99,7 @@ steps: - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-minimum-services - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service bitwarden - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service gitea - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service jitsi # - FIXME it should be jitsi-meet + - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service jitsi-meet - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service ocserv - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service pleroma - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#nixos-rebuild Rebuild From 149d6a8dd2ddfc0b220810613f26f8c4c2561c84 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 17:08:54 +0400 Subject: [PATCH 076/124] 1000 seconds sleep on failure while machine pings --- .drone.yml | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.drone.yml b/.drone.yml index aaeaed9..0640b86 100644 --- a/.drone.yml +++ b/.drone.yml @@ -76,7 +76,7 @@ steps: - name: sleep commands: - - sleep 420 + - sleep 360 - name: test environment: @@ -88,6 +88,8 @@ steps: from_secret: TEST_EMAIL_PASS API_TOKEN: from_secret: USER_PASS + TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git + commands: - dig api.$DOMAIN - http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" @@ -96,22 +98,27 @@ steps: - echo "File to test nextcloud upload" > file - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-minimum-services - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service bitwarden - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service gitea - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service jitsi-meet - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service ocserv - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#enable-service pleroma - - nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#nixos-rebuild Rebuild - - counter=1; while true; do sleep 4; nix --extra-experimental-features "nix-command flakes" run git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git#query-all-services && break; [ "$counter" -le 30 ]; ((counter++)); done + - nix run $TESTS_REPO#query-minimum-services + - nix run $TESTS_REPO#enable-service bitwarden + - nix run $TESTS_REPO#enable-service gitea + - nix run $TESTS_REPO#enable-service jitsi-meet + - nix run $TESTS_REPO#enable-service ocserv + - nix run $TESTS_REPO#enable-service pleroma + - nix run $TESTS_REPO#nixos-rebuild Rebuild + - counter=0; while true; do sleep 4; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 30 ]; ((++counter)); done + - nix run $TESTS_REPO#nixos-rebuild Rollback + - counter=0; while true; do sleep 4; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 30 ]; ((++counter)); done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics -# - name: sleep-after-failure -# commands: -# - sleep 240 -# when: -# status: -# - failure +- name: sleep-after-failure + environment: + DOMAIN: + from_secret: DOMAIN + commands: + - counter=0; while [ "$counter" -lt 100 ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((++counter)); done + when: + status: + - failure - name: teardown environment: From d4e33a9639e72c54d09aa69306a47f7750d656cf Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 12 Dec 2023 20:37:39 +0400 Subject: [PATCH 077/124] notifications during retries --- .drone.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0640b86..6e1dd19 100644 --- a/.drone.yml +++ b/.drone.yml @@ -105,9 +105,11 @@ steps: - nix run $TESTS_REPO#enable-service ocserv - nix run $TESTS_REPO#enable-service pleroma - nix run $TESTS_REPO#nixos-rebuild Rebuild - - counter=0; while true; do sleep 4; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 30 ]; ((++counter)); done + - sleep 60 + - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining to retry"; done - nix run $TESTS_REPO#nixos-rebuild Rollback - - counter=0; while true; do sleep 4; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 30 ]; ((++counter)); done + - sleep 10 + - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining to retry"; done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure @@ -115,7 +117,7 @@ steps: DOMAIN: from_secret: DOMAIN commands: - - counter=0; while [ "$counter" -lt 100 ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((++counter)); done + - counter=0; while [ "$counter" -lt 100 ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((++counter)); echo "$((100 * 10 - counter * 10)) seconds remaining before teardown - login via ssh to shutdown the machine"; done when: status: - failure From f3e19457ee314f7b189068751076084d4a46ea0c Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 16 Dec 2023 09:44:02 +0400 Subject: [PATCH 078/124] do not write resticPassword to secrets.json --- nixos-infect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos-infect b/nixos-infect index 87c165b..1427d59 100755 --- a/nixos-infect +++ b/nixos-infect @@ -73,8 +73,7 @@ genSecrets() { "adminPassword": "$USER_PASS", "databasePassword": "$USER_PASS" } - }, - "resticPassword": "$USER_PASS" + } } EOF } From 0e44c12bef50ba990b7df66cc2e3f61e98efa6bf Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sun, 17 Dec 2023 16:41:44 +0400 Subject: [PATCH 079/124] retries message --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6e1dd19..4599e0a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -106,10 +106,10 @@ steps: - nix run $TESTS_REPO#enable-service pleroma - nix run $TESTS_REPO#nixos-rebuild Rebuild - sleep 60 - - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining to retry"; done + - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining for retries"; done - nix run $TESTS_REPO#nixos-rebuild Rollback - sleep 10 - - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining to retry"; done + - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining for retries"; done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure From efad17f1947214c8fd443d3f0517ee7c75cc6153 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 18 Dec 2023 13:28:11 +0400 Subject: [PATCH 080/124] wait minimal services startup for 1000 seconds --- .drone.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 4599e0a..e156e74 100644 --- a/.drone.yml +++ b/.drone.yml @@ -89,6 +89,8 @@ steps: API_TOKEN: from_secret: USER_PASS TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git + TIMEOUT_1: 100 + TIMEOUT_2: 1000 commands: - dig api.$DOMAIN @@ -106,18 +108,19 @@ steps: - nix run $TESTS_REPO#enable-service pleroma - nix run $TESTS_REPO#nixos-rebuild Rebuild - sleep 60 - - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining for retries"; done + - timer=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; ((timer+=10)); echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; done - nix run $TESTS_REPO#nixos-rebuild Rollback - sleep 10 - - counter=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$counter" -lt 10 ]; ((++counter)); echo "$((10 * 10 - counter * 10)) seconds remaining for retries"; done + - timer=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_2" ]; ((timer+=10)); echo "$((TIMEOUT_2 - timer)) seconds remain for retries"; done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure environment: DOMAIN: from_secret: DOMAIN + TIMEOUT: 1000 commands: - - counter=0; while [ "$counter" -lt 100 ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((++counter)); echo "$((100 * 10 - counter * 10)) seconds remaining before teardown - login via ssh to shutdown the machine"; done + - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((timer+=10)); echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; done when: status: - failure From e3ad189d9a7f6d5ba6e070cc127581afb0525c35 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 18 Dec 2023 16:00:06 +0400 Subject: [PATCH 081/124] add retries for query-minimum-services --- .drone.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index e156e74..aa2a6c7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -100,7 +100,7 @@ steps: - echo "File to test nextcloud upload" > file - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - - nix run $TESTS_REPO#query-minimum-services + - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#enable-service bitwarden - nix run $TESTS_REPO#enable-service gitea - nix run $TESTS_REPO#enable-service jitsi-meet @@ -108,10 +108,10 @@ steps: - nix run $TESTS_REPO#enable-service pleroma - nix run $TESTS_REPO#nixos-rebuild Rebuild - sleep 60 - - timer=0; while true; do sleep 10; nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; ((timer+=10)); echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; done + - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#nixos-rebuild Rollback - sleep 10 - - timer=0; while true; do sleep 10; nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_2" ]; ((timer+=10)); echo "$((TIMEOUT_2 - timer)) seconds remain for retries"; done + - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_2" ]; echo "$((TIMEOUT_2 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure @@ -120,7 +120,7 @@ steps: from_secret: DOMAIN TIMEOUT: 1000 commands: - - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do sleep 10; nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; ((timer+=10)); echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; done + - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done when: status: - failure From 7d1546086739edb473d8e506916a42c3fbc0ec14 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 18 Dec 2023 16:31:57 +0400 Subject: [PATCH 082/124] slightly increase retry timeout after nixos-rebuild switch --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index aa2a6c7..66cafcd 100644 --- a/.drone.yml +++ b/.drone.yml @@ -107,7 +107,7 @@ steps: - nix run $TESTS_REPO#enable-service ocserv - nix run $TESTS_REPO#enable-service pleroma - nix run $TESTS_REPO#nixos-rebuild Rebuild - - sleep 60 + - sleep 70 - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#nixos-rebuild Rollback - sleep 10 From 7099e2eada8c63f17bbf1241667a5f1b991ec430 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 18 Dec 2023 16:56:21 +0400 Subject: [PATCH 083/124] re-enable all services after rollback --- .drone.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 66cafcd..b5ba978 100644 --- a/.drone.yml +++ b/.drone.yml @@ -90,7 +90,6 @@ steps: from_secret: USER_PASS TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git TIMEOUT_1: 100 - TIMEOUT_2: 1000 commands: - dig api.$DOMAIN @@ -111,7 +110,15 @@ steps: - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#nixos-rebuild Rollback - sleep 10 - - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_2" ]; echo "$((TIMEOUT_2 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done + - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done + - nix run $TESTS_REPO#enable-service bitwarden + - nix run $TESTS_REPO#enable-service gitea + - nix run $TESTS_REPO#enable-service jitsi-meet + - nix run $TESTS_REPO#enable-service ocserv + - nix run $TESTS_REPO#enable-service pleroma + - nix run $TESTS_REPO#nixos-rebuild Rebuild + - sleep 60 + - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure From 0857b960014377cb34e0ed522fd47d02c7075d3b Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Mon, 18 Dec 2023 23:57:50 +0400 Subject: [PATCH 084/124] STAGING_ACME=false --- .drone.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.drone.yml b/.drone.yml index b5ba978..6683305 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns @@ -164,11 +164,3 @@ steps: status: - failure - success - -- name: switch - environment: - CHANNEL_SWITCHER_KEY: - from_secret: CHANNEL_SWITCHER_KEY - commands: - - > - curl -X POST -H "Authorization: $CHANNEL_SWITCHER_KEY" 'https://channel.selfprivacy.org/nixos-selfpricacy/upgrade' From 0e6d5414f5a3b885eb100e892a278f610b877ba8 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 20 Dec 2023 16:36:14 +0400 Subject: [PATCH 085/124] CF_TOKEN => DNS_PROVIDER_TOKEN --- .drone.yml | 2 +- nixos-infect | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 6683305..f6b8abf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" CF_TOKEN=$CLOUDFLARE_TOKEN DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test DNS_PROVIDER_TYPE=CLOUDFLARE STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns diff --git a/nixos-infect b/nixos-infect index 1427d59..eb4daab 100755 --- a/nixos-infect +++ b/nixos-infect @@ -11,7 +11,7 @@ : "${PROVIDER:?PROVIDER variable is not set}" : "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}" : "${STAGING_ACME:?STAGING_ACME variable is not set}" -: "${CF_TOKEN:?CF_TOKEN variable is not set}" +: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" : "${DB_PASSWORD:?DB_PASSWORD variable is not set}" : "${USER_PASS:?USER_PASS variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" @@ -66,7 +66,7 @@ genSecrets() { }, "databasePassword": "$DB_PASSWORD", "dns": { - "apiKey": "$CF_TOKEN" + "apiKey": "$DNS_PROVIDER_TOKEN" }, "modules": { "nextcloud": { From 1344aaa92cc6c784d53cec81d9e3819e6e65e2e0 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 20 Dec 2023 16:41:42 +0400 Subject: [PATCH 086/124] `nix flake lock --override-input sp-modules` instead of --update-input --- nixos-infect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos-infect b/nixos-infect index eb4daab..68f211e 100755 --- a/nixos-infect +++ b/nixos-infect @@ -423,7 +423,7 @@ infect() { # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ --extra-experimental-features "nix-command flakes" \ - --update-input sp-modules + --override-input sp-modules ${LOCAL_FLAKE_DIR}/sp-modules echo "nix build the configuration flake..." if ! /root/.nix-profile/bin/nix build \ From a717ad6ca532ebfdc468c6c194c1ffdc9dda55fe Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 20 Dec 2023 17:39:49 +0400 Subject: [PATCH 087/124] STAGING_ACME=true --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index f6b8abf..8761c30 100644 --- a/.drone.yml +++ b/.drone.yml @@ -21,7 +21,7 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test STAGING_ACME=false NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns From 4380a733a62de42d17ba77c1030260e83fea46cc Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 20 Dec 2023 18:50:26 +0400 Subject: [PATCH 088/124] wait for 443 port instead of sleep --- .drone.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8761c30..de3dc0d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -74,9 +74,14 @@ steps: -H "Content-Type: application/json" --data '{"type":"TXT","name":"'"$$DOMAIN"'","content":"v=spf1 a mx ip4:$machineip -all","ttl":18000,"priority":10,"proxied":false}' -- name: sleep +- name: wait for NixOS boot + environment: + DOMAIN: + from_secret: DOMAIN + TIMEOUT: 100 commands: - - sleep 360 + - sleep 300 + - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done - name: test environment: From 3b4d75bcdb494230828d61ac1c0dab13c105d6ab Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 20 Dec 2023 21:43:47 +0400 Subject: [PATCH 089/124] use new retry function; split test into 2 --- .drone.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index de3dc0d..003eb68 100644 --- a/.drone.yml +++ b/.drone.yml @@ -80,10 +80,11 @@ steps: from_secret: DOMAIN TIMEOUT: 100 commands: - - sleep 300 - - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done + - sleep 300 + - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done + - sleep 10 -- name: test +- name: basic test environment: DOMAIN: from_secret: DOMAIN @@ -95,15 +96,26 @@ steps: from_secret: USER_PASS TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git TIMEOUT_1: 100 - commands: + - source $(nix build $TESTS_REPO#library --print-out-paths)/library.sh - dig api.$DOMAIN - - http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - retry $TIMEOUT_1 10 http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - echo "File to test nextcloud upload" > file - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS + #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics + +- name: API & nixos-rebuild test + environment: + DOMAIN: + from_secret: DOMAIN + API_TOKEN: + from_secret: USER_PASS + TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git + TIMEOUT_1: 100 + commands: - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#enable-service bitwarden - nix run $TESTS_REPO#enable-service gitea @@ -124,7 +136,6 @@ steps: - nix run $TESTS_REPO#nixos-rebuild Rebuild - sleep 60 - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics - name: sleep-after-failure environment: From 09ee1f5e4c84d3ccf2e424e45b5c0357cea1b74e Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 21 Dec 2023 12:18:13 +0400 Subject: [PATCH 090/124] 3 times increase sleep after failure timeout --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 003eb68..5529ca9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -137,11 +137,11 @@ steps: - sleep 60 - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done -- name: sleep-after-failure +- name: sleep after failure environment: DOMAIN: from_secret: DOMAIN - TIMEOUT: 1000 + TIMEOUT: 3000 commands: - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done when: From 30f5d85acdaa1be3b149fa2734daff620391753b Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 21 Dec 2023 12:35:59 +0400 Subject: [PATCH 091/124] fix ip4 value for domain TXT DNS record --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5529ca9..5288fbf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -72,7 +72,7 @@ steps: curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json" - --data '{"type":"TXT","name":"'"$$DOMAIN"'","content":"v=spf1 a mx ip4:$machineip -all","ttl":18000,"priority":10,"proxied":false}' + --data "{\"type\":\"TXT\",\"name\":\"$DOMAIN\",\"content\":\"v=spf1 a mx ip4:$machineip -all\",\"ttl\":18000,\"priority\":10,\"proxied\":false}" - name: wait for NixOS boot environment: From 55cf8bc0a6439e9d596761541f45cee5d2cafd2d Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 21 Dec 2023 13:19:57 +0400 Subject: [PATCH 092/124] wait for NixOS boot 460 seconds at maximum --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 5288fbf..cae9fed 100644 --- a/.drone.yml +++ b/.drone.yml @@ -78,7 +78,7 @@ steps: environment: DOMAIN: from_secret: DOMAIN - TIMEOUT: 100 + TIMEOUT: 160 commands: - sleep 300 - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done From ec339bb8d151e088e4995d48ea840d0166134e44 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 22 Dec 2023 14:03:50 +0400 Subject: [PATCH 093/124] use tests library for `enable-service` tests --- .drone.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.drone.yml b/.drone.yml index cae9fed..a2ab856 100644 --- a/.drone.yml +++ b/.drone.yml @@ -97,7 +97,7 @@ steps: TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git TIMEOUT_1: 100 commands: - - source $(nix build $TESTS_REPO#library --print-out-paths)/library.sh + - source $(nix build $TESTS_REPO#library --print-out-paths) - dig api.$DOMAIN - retry $TIMEOUT_1 10 http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp @@ -116,13 +116,14 @@ steps: TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git TIMEOUT_1: 100 commands: - - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - - nix run $TESTS_REPO#enable-service bitwarden - - nix run $TESTS_REPO#enable-service gitea - - nix run $TESTS_REPO#enable-service jitsi-meet - - nix run $TESTS_REPO#enable-service ocserv - - nix run $TESTS_REPO#enable-service pleroma - - nix run $TESTS_REPO#nixos-rebuild Rebuild + - source $(nix build $TESTS_REPO#library --print-out-paths) + - retry $TIMEOUT_1 10 query-minimum-services + - enable-service bitwarden + - enable-service gitea + - enable-service jitsi-meet + - enable-service ocserv + - enable-service pleroma + - nixos-rebuild Rebuild - sleep 70 - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - nix run $TESTS_REPO#nixos-rebuild Rollback From 6f96b9afdf2d3939ecd5919db65350ac4ea45b4a Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 22 Dec 2023 14:43:13 +0400 Subject: [PATCH 094/124] add sleep 1 before nixos-rebuild API request --- .drone.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.drone.yml b/.drone.yml index a2ab856..90834ab 100644 --- a/.drone.yml +++ b/.drone.yml @@ -123,20 +123,23 @@ steps: - enable-service jitsi-meet - enable-service ocserv - enable-service pleroma + - sleep 1 - nixos-rebuild Rebuild - sleep 70 - - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - - nix run $TESTS_REPO#nixos-rebuild Rollback + - retry $TIMEOUT_1 10 $TESTS_REPO#query-all-services + - sleep 1 + - nixos-rebuild Rollback - sleep 10 - - timer=0; while true; do nix run $TESTS_REPO#query-minimum-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done - - nix run $TESTS_REPO#enable-service bitwarden - - nix run $TESTS_REPO#enable-service gitea - - nix run $TESTS_REPO#enable-service jitsi-meet - - nix run $TESTS_REPO#enable-service ocserv - - nix run $TESTS_REPO#enable-service pleroma - - nix run $TESTS_REPO#nixos-rebuild Rebuild + - retry $TIMEOUT_1 10 query-minimum-services + - enable-service bitwarden + - enable-service gitea + - enable-service jitsi-meet + - enable-service ocserv + - enable-service pleroma + - sleep 1 + - nixos-rebuild Rebuild - sleep 60 - - timer=0; while true; do nix run $TESTS_REPO#query-all-services && break; [ "$timer" -lt "$TIMEOUT_1" ]; echo "$((TIMEOUT_1 - timer)) seconds remain for retries"; sleep 10; ((timer+=10)); done + - retry $TIMEOUT_1 10 query-all-services - name: sleep after failure environment: From 2d5465392455e9a252a51087a0fabb192ec67285 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 22 Dec 2023 21:32:43 +0400 Subject: [PATCH 095/124] ci: retry first request to nextcloud --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 90834ab..99e4822 100644 --- a/.drone.yml +++ b/.drone.yml @@ -101,7 +101,7 @@ steps: - dig api.$DOMAIN - retry $TIMEOUT_1 10 http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - retry 30 10 http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - echo "File to test nextcloud upload" > file - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS From 9cd620723810ab14bf7cbb925884c0fb9ab1df82 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 22 Dec 2023 21:33:07 +0400 Subject: [PATCH 096/124] add stateVersion to userdata.json --- nixos-infect | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/nixos-infect b/nixos-infect index 68f211e..c540c61 100755 --- a/nixos-infect +++ b/nixos-infect @@ -22,6 +22,8 @@ readonly NL=$'\n' readonly LOCAL_FLAKE_DIR="/etc/nixos" readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json" +readonly NIX="/root/.nix-profile/bin/nix" +readonly NIX_OPTS=(--extra-experimental-features "nix-command flakes") DoNetConf= genOptionalSsh() { @@ -32,9 +34,15 @@ EOF # Merge original userdata.json with deployment specific fields and print result. genUserdata() { - local HASHED_PASSWORD userdata_infect + local HASHED_PASSWORD userdata_infect release HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")" + if ! release="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr "(builtins.getFlake (builtins.toString ${LOCAL_FLAKE_DIR})).inputs.selfprivacy-nixos-config.inputs.nixpkgs.lib.trivial.release")" + then + echo "cannot determine NixOS release version" + return 1 + fi + userdata_infect=$(cat << EOF { $(genOptionalSsh) @@ -48,7 +56,8 @@ genUserdata() { "domain": "$DOMAIN", "hashedMasterPassword": "$HASHED_PASSWORD", "hostname": "$HOSTNAME", - "username": "$LUSER" + "username": "$LUSER", + "stateVersion": "$release" } EOF ) @@ -97,8 +106,6 @@ EOF ) fi - local NIX="/root/.nix-profile/bin/nix" - local NIX_OPTS=(--extra-experimental-features nix-command) local currentSystem if ! currentSystem="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr builtins.currentSystem)" then @@ -132,8 +139,6 @@ EOF } genDeploymentConfiguration() { - local NIX="/root/.nix-profile/bin/nix" - local NIX_OPTS=(--extra-experimental-features "nix-command flakes") local release if ! release="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr "(builtins.getFlake (builtins.toString ${LOCAL_FLAKE_DIR})).inputs.selfprivacy-nixos-config.inputs.nixpkgs.lib.trivial.release")" @@ -147,7 +152,7 @@ ${DoNetConf:+"{ lib, ... }: "}{ # The content below is static and belongs to this deployment only! # Do not copy this configuration file to another NixOS installation! - system.stateVersion = "$release";` + # system.stateVersion = "$release";` `$(if [ "$DoNetConf" == "y" ]; then echo -e "$NL"; genNetworkingConf; fi) } EOF From d099b7bd11febec9d9467918c0995e6b4fe70e47 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Fri, 22 Dec 2023 23:04:38 +0400 Subject: [PATCH 097/124] system.stateVersion = lib.mkDefault "$release" --- nixos-infect | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/nixos-infect b/nixos-infect index c540c61..1208c0c 100755 --- a/nixos-infect +++ b/nixos-infect @@ -37,12 +37,6 @@ genUserdata() { local HASHED_PASSWORD userdata_infect release HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")" - if ! release="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr "(builtins.getFlake (builtins.toString ${LOCAL_FLAKE_DIR})).inputs.selfprivacy-nixos-config.inputs.nixpkgs.lib.trivial.release")" - then - echo "cannot determine NixOS release version" - return 1 - fi - userdata_infect=$(cat << EOF { $(genOptionalSsh) @@ -56,8 +50,7 @@ genUserdata() { "domain": "$DOMAIN", "hashedMasterPassword": "$HASHED_PASSWORD", "hostname": "$HOSTNAME", - "username": "$LUSER", - "stateVersion": "$release" + "username": "$LUSER" } EOF ) @@ -148,11 +141,11 @@ genDeploymentConfiguration() { fi cat << EOF -${DoNetConf:+"{ lib, ... }: "}{ +{ lib, ... }: { # The content below is static and belongs to this deployment only! # Do not copy this configuration file to another NixOS installation! - # system.stateVersion = "$release";` + system.stateVersion = lib.mkDefault "$release";` `$(if [ "$DoNetConf" == "y" ]; then echo -e "$NL"; genNetworkingConf; fi) } EOF From ec40186962d27195bb48201d708c088c861c2763 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 23 Dec 2023 00:14:04 +0400 Subject: [PATCH 098/124] ci: enable SSL certificate verification in basic test --- .drone.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index 99e4822..209cb2b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -99,13 +99,13 @@ steps: commands: - source $(nix build $TESTS_REPO#library --print-out-paths) - dig api.$DOMAIN - - retry $TIMEOUT_1 10 http -v --verify=false --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - sleep 1 + - retry $TIMEOUT_1 10 http -v --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - retry 30 10 http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - retry 30 10 http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - echo "File to test nextcloud upload" > file - - http -v --verify=false --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - - http -v --verify=false --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - #- > #http -v --verify=false --check-status --ignore-stdin GET https://password.$DOMAIN/admin/diagnostics + - http -v --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS + - http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - name: API & nixos-rebuild test environment: From dcfc5f203ef7271db40190c0a6fa7ddc95440bf0 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 23 Dec 2023 00:16:16 +0400 Subject: [PATCH 099/124] ci: no unnecessary sleep and retry in basic test --- .drone.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 209cb2b..e19f768 100644 --- a/.drone.yml +++ b/.drone.yml @@ -99,10 +99,9 @@ steps: commands: - source $(nix build $TESTS_REPO#library --print-out-paths) - dig api.$DOMAIN - - sleep 1 - retry $TIMEOUT_1 10 http -v --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - retry 30 10 http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - echo "File to test nextcloud upload" > file - http -v --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS From 1494e06c44c0b5e6ba3fcf5821ada7ff67260acd Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 23 Dec 2023 00:17:53 +0400 Subject: [PATCH 100/124] ci: fix typo for starting query-all-services --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index e19f768..c6632f7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -125,7 +125,7 @@ steps: - sleep 1 - nixos-rebuild Rebuild - sleep 70 - - retry $TIMEOUT_1 10 $TESTS_REPO#query-all-services + - retry $TIMEOUT_1 10 query-all-services - sleep 1 - nixos-rebuild Rollback - sleep 10 From ab35fc20b69dcbc958b97aa70d02fb0880916ff1 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 23 Dec 2023 00:36:13 +0400 Subject: [PATCH 101/124] ci: SSL certificate verification in basic test based on STAGING_ACME --- .drone.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index c6632f7..984e64a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -20,8 +20,13 @@ steps: commands: # Create infect user script and then push it to a remote machine on server creation. - - echo '#! /usr/bin/env bash' > infect.sh - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test STAGING_ACME=true NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh + - | + cat << EOF > infect.sh + #! /usr/bin/env bash + + export STAGING_ACME=false + EOF + - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns @@ -98,13 +103,16 @@ steps: TIMEOUT_1: 100 commands: - source $(nix build $TESTS_REPO#library --print-out-paths) + # ideally it should depend on $STAGING_ACME + # VERIFY=$(if [[ $STAGING_ACME == true ]]; then echo "no"; else echo "yes"; fi) + - VERIFY="on" - dig api.$DOMAIN - - retry $TIMEOUT_1 10 http -v --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - retry $TIMEOUT_1 10 http -v --verify=$VERIFY --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - http -v --verify=$VERIFY --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS - echo "File to test nextcloud upload" > file - - http -v --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - - http -v --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS + - http -v --verify=$VERIFY --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS + - http -v --verify=$VERIFY --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS - name: API & nixos-rebuild test environment: From b135e528634dd4f720ed18efc893762e5c0cf1ff Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sat, 23 Dec 2023 11:04:59 +0400 Subject: [PATCH 102/124] replace HTTPie with curl in basic test --- .drone.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 984e64a..dc00a86 100644 --- a/.drone.yml +++ b/.drone.yml @@ -102,17 +102,18 @@ steps: TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git TIMEOUT_1: 100 commands: - - source $(nix build $TESTS_REPO#library --print-out-paths) # ideally it should depend on $STAGING_ACME - # VERIFY=$(if [[ $STAGING_ACME == true ]]; then echo "no"; else echo "yes"; fi) - - VERIFY="on" + # INSECURE=$(if [[ $STAGING_ACME == true ]]; then echo "--insecure"; else echo ""; fi) + - INSECURE="--insecure" + - source $(nix build $TESTS_REPO#library --print-out-paths) - dig api.$DOMAIN - - retry $TIMEOUT_1 10 http -v --verify=$VERIFY --check-status --ignore-stdin GET https://api.$DOMAIN/services/status Authorization:"Bearer $USER_PASS" + - > + retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $INSECURE -H "Authorization: Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - http -v --verify=$VERIFY --check-status --ignore-stdin GET https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -a admin:$USER_PASS + - curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json --fail-with-body $INSECURE -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"' - echo "File to test nextcloud upload" > file - - http -v --verify=$VERIFY --check-status --ignore-stdin PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file @file -a admin:$USER_PASS - - http -v --verify=$VERIFY --check-status --ignore-stdin GET https://cloud.$DOMAIN/remote.php/dav/files/admin/file -a admin:$USER_PASS + - curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -d @file -u "admin:$USER_PASS" + - curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -u "admin:$USER_PASS" - name: API & nixos-rebuild test environment: From a266532d59f62490654f45c14e11801b50f77f58 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sun, 24 Dec 2023 07:32:09 +0400 Subject: [PATCH 103/124] remove unnecessary sleeps before API queries --- .drone.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.drone.yml b/.drone.yml index dc00a86..d8e6e83 100644 --- a/.drone.yml +++ b/.drone.yml @@ -131,20 +131,17 @@ steps: - enable-service jitsi-meet - enable-service ocserv - enable-service pleroma - - sleep 1 - nixos-rebuild Rebuild - - sleep 70 + - sleep 100 - retry $TIMEOUT_1 10 query-all-services - - sleep 1 - nixos-rebuild Rollback - - sleep 10 + - sleep 30 - retry $TIMEOUT_1 10 query-minimum-services - enable-service bitwarden - enable-service gitea - enable-service jitsi-meet - enable-service ocserv - enable-service pleroma - - sleep 1 - nixos-rebuild Rebuild - sleep 60 - retry $TIMEOUT_1 10 query-all-services From 6171589cc3412f4154a31c9be604e9d8d1185a35 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sun, 24 Dec 2023 07:35:39 +0400 Subject: [PATCH 104/124] ci: curl commands in basic test start with `>` --- .drone.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index d8e6e83..8a12415 100644 --- a/.drone.yml +++ b/.drone.yml @@ -110,10 +110,13 @@ steps: - > retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $INSECURE -H "Authorization: Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - - curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json --fail-with-body $INSECURE -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"' + - > + curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json --fail-with-body $INSECURE -u "admin:$USER_PASS" | jq -re ".ocs.meta.status == \"ok\"" - echo "File to test nextcloud upload" > file - - curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -d @file -u "admin:$USER_PASS" - - curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -u "admin:$USER_PASS" + - > + curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -d @file -u "admin:$USER_PASS" + - > + curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -u "admin:$USER_PASS" - name: API & nixos-rebuild test environment: From 867396b274646326b8fb008e88bced8058f3c971 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Sun, 24 Dec 2023 15:49:35 +0400 Subject: [PATCH 105/124] use staging ACME with root CA verification --- .drone.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8a12415..75f5ae7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -24,7 +24,7 @@ steps: cat << EOF > infect.sh #! /usr/bin/env bash - export STAGING_ACME=false + export STAGING_ACME=true EOF - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 @@ -87,7 +87,8 @@ steps: commands: - sleep 300 - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done - - sleep 10 + - echo "give time for obtaining ACME certificate" + - sleep 60 - name: basic test environment: @@ -103,20 +104,21 @@ steps: TIMEOUT_1: 100 commands: # ideally it should depend on $STAGING_ACME - # INSECURE=$(if [[ $STAGING_ACME == true ]]; then echo "--insecure"; else echo ""; fi) - - INSECURE="--insecure" + # CERT_VERIFY=$(if [[ $STAGING_ACME == true ]]; then echo "--cacert root.pem"; else echo ""; fi) + - CERT_VERIFY="--cacert root.pem" - source $(nix build $TESTS_REPO#library --print-out-paths) + - curl https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -sS --fail -o root.pem - dig api.$DOMAIN - > - retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $INSECURE -H "Authorization: Bearer $USER_PASS" + retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - > - curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json --fail-with-body $INSECURE -u "admin:$USER_PASS" | jq -re ".ocs.meta.status == \"ok\"" - - echo "File to test nextcloud upload" > file + curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"' + - echo 'File to test nextcloud upload' > file - > - curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -d @file -u "admin:$USER_PASS" + curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -d @file -u "admin:$USER_PASS" - > - curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file --fail-with-body $INSECURE -u "admin:$USER_PASS" + curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" - name: API & nixos-rebuild test environment: From 69dd80d0652d512f744ef369409e04c09aa348b3 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 26 Dec 2023 13:47:35 +0400 Subject: [PATCH 106/124] more readable nixos-infect command; DRONE_COMMIT_SHA --- .drone.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/.drone.yml b/.drone.yml index 75f5ae7..39fff14 100644 --- a/.drone.yml +++ b/.drone.yml @@ -16,7 +16,7 @@ steps: from_secret: CLOUDFLARE_TOKEN USER_PASS: from_secret: USER_PASS - INFECT_SOURCE_BRANCH: ${DRONE_SOURCE_BRANCH} + INFECT_COMMIT_SHA: ${DRONE_COMMIT_SHA} commands: # Create infect user script and then push it to a remote machine on server creation. @@ -24,9 +24,29 @@ steps: cat << EOF > infect.sh #! /usr/bin/env bash - export STAGING_ACME=true + set -o allexport + set -o pipefail + set -o errexit + + API_TOKEN="$USER_PASS" + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz + DB_PASSWORD="$USER_PASS" + DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN + DNS_PROVIDER_TYPE=CLOUDFLARE + DOMAIN=$DOMAIN + HOSTNAME=selfprivacy-ci-test + LUSER=cicdcicd + NIXOS_CONFIG_NAME=sp-nixos + NIX_VERSION=2.18.1 + PROVIDER=hetzner + SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K" + STAGING_ACME=true + USER_PASS="$USER_PASS" + + curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \ + | bash 2>&1 | tee /root/nixos-infect.log EOF - - echo "curl https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/branch/$INFECT_SOURCE_BRANCH/nixos-infect | PROVIDER=hetzner DOMAIN=$DOMAIN LUSER=cicdcicd USER_PASS=\"$USER_PASS\" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DB_PASSWORD=\"$USER_PASS\" API_TOKEN=\"$USER_PASS\" HOSTNAME=selfprivacy-ci-test NIX_VERSION=2.18.1 NIXOS_CONFIG_NAME=sp-nixos CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz SSH_AUTHORIZED_KEY=\"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K\" bash 2>&1 | tee /root/infect.log" >> infect.sh + - cat infect.sh - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 - name: dns From eec1c639ece3c301aa186ecb132bfa0cda2befc4 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 26 Dec 2023 18:53:16 +0400 Subject: [PATCH 107/124] reduce sleep after failure time to 1500 seconds --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 39fff14..2b8f363 100644 --- a/.drone.yml +++ b/.drone.yml @@ -175,7 +175,7 @@ steps: environment: DOMAIN: from_secret: DOMAIN - TIMEOUT: 3000 + TIMEOUT: 1500 commands: - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done when: From a6713df9314e26fca9f1873c3d4666c4c8ac400e Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 26 Dec 2023 18:52:12 +0400 Subject: [PATCH 108/124] pin selfprivacy-nixos-template commit --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 2b8f363..1e8b619 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: set -o errexit API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/master.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/eb9aebc8ace3f82748dc325374058df534056420.tar.gz DB_PASSWORD="$USER_PASS" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE From 05e395d6af0bdce0c6c66ff4a3d157cad9dc1100 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 13:10:35 +0400 Subject: [PATCH 109/124] nix flake lock /etc/nixos --update-input sp-modules --- nixos-infect | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/nixos-infect b/nixos-infect index 1208c0c..ab2f037 100755 --- a/nixos-infect +++ b/nixos-infect @@ -417,15 +417,12 @@ installNix() { } infect() { - # this is needed solely for accepting the sp-module subflake + # this is needed solely for accepting the sp-module sub-flake # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 - /root/.nix-profile/bin/nix flake lock ${LOCAL_FLAKE_DIR} \ - --extra-experimental-features "nix-command flakes" \ - --override-input sp-modules ${LOCAL_FLAKE_DIR}/sp-modules + $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} --update-input sp-modules echo "nix build the configuration flake..." - if ! /root/.nix-profile/bin/nix build \ - --extra-experimental-features "nix-command flakes" \ + if ! $NIX "${NIX_OPTS[@]}" build \ --profile /nix/var/nix/profiles/system \ ${LOCAL_FLAKE_DIR}/#nixosConfigurations.sp-nixos.config.system.build.toplevel then From c95b9d0f62178cb6afdf5324d662108122808fd2 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 14:05:09 +0400 Subject: [PATCH 110/124] /etc/nixos#sp-nixos => /etc/nixos#default --- .drone.yml | 4 ++-- nixos-infect | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index 1e8b619..74ecf02 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,14 +29,14 @@ steps: set -o errexit API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/eb9aebc8ace3f82748dc325374058df534056420.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/58d43eec2f69c8fada5eea3e20b917de26bf8bb6.tar.gz DB_PASSWORD="$USER_PASS" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN HOSTNAME=selfprivacy-ci-test LUSER=cicdcicd - NIXOS_CONFIG_NAME=sp-nixos + NIXOS_CONFIG_ID=default NIX_VERSION=2.18.1 PROVIDER=hetzner SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K" diff --git a/nixos-infect b/nixos-infect index ab2f037..01e4ef7 100755 --- a/nixos-infect +++ b/nixos-infect @@ -15,7 +15,7 @@ : "${DB_PASSWORD:?DB_PASSWORD variable is not set}" : "${USER_PASS:?USER_PASS variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" -: "${NIXOS_CONFIG_NAME:?NIXOS_CONFIG_NAME variable is not set}" +: "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" : "${SSH_AUTHORIZED_KEY:=}" @@ -424,7 +424,7 @@ infect() { echo "nix build the configuration flake..." if ! $NIX "${NIX_OPTS[@]}" build \ --profile /nix/var/nix/profiles/system \ - ${LOCAL_FLAKE_DIR}/#nixosConfigurations.sp-nixos.config.system.build.toplevel + ${LOCAL_FLAKE_DIR}/#nixosConfigurations."$NIXOS_CONFIG_ID".config.system.build.toplevel then echo "Failed!" exit 1 From 4df84db4531b7a1b41423a48c58b48c8c8e5b330 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 14:13:40 +0400 Subject: [PATCH 111/124] curl --fail --- nixos-infect | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos-infect b/nixos-infect index 01e4ef7..3d24392 100755 --- a/nixos-infect +++ b/nixos-infect @@ -153,7 +153,7 @@ EOF setupConf() { mkdir -p ${LOCAL_FLAKE_DIR} - if ! curl "${CONFIG_URL}" \ + if ! curl --fail "${CONFIG_URL}" \ | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 --exclude=".*" then echo "Error downloading/extracting top level flake configuration!" @@ -389,12 +389,12 @@ installNix() { fi echo "Downloading install script from ${installURL}..." - if ! curl "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then + if ! curl --fail "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then echo "Failure while downloading Nix install script!" return 1 fi - if ! sha="$(curl "${shaURL}")"; then + if ! sha="$(curl --fail "${shaURL}")"; then echo "Failure while downloading Nix install script sha!" return 1 fi From 659c539f38bf4a13fb3b2e8437322109ad180cd8 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 15:17:31 +0400 Subject: [PATCH 112/124] update --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 74ecf02..4d9eadb 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,7 +29,7 @@ steps: set -o errexit API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/58d43eec2f69c8fada5eea3e20b917de26bf8bb6.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/0f886d76e93dd366db7c53a8f6b672702910b99b.tar.gz DB_PASSWORD="$USER_PASS" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE From 967377f1719907777df5d3e83b505557362401f8 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 17:59:09 +0400 Subject: [PATCH 113/124] pass ENCODED_PASSWORD to nixos-infect instead of USER_PASS --- .drone.yml | 5 ++++- nixos-infect | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 4d9eadb..5b6a5d1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -19,6 +19,9 @@ steps: INFECT_COMMIT_SHA: ${DRONE_COMMIT_SHA} commands: + - set -o nounset + - > # TODO pass Base64 encoded password from Drone instead of this + ENCODED_PASSWORD="$(base64 <<<"$USER_PASS")" # Create infect user script and then push it to a remote machine on server creation. - | cat << EOF > infect.sh @@ -34,6 +37,7 @@ steps: DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN + ENCODED_PASSWORD="$ENCODED_PASSWORD" HOSTNAME=selfprivacy-ci-test LUSER=cicdcicd NIXOS_CONFIG_ID=default @@ -41,7 +45,6 @@ steps: PROVIDER=hetzner SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K" STAGING_ACME=true - USER_PASS="$USER_PASS" curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \ | bash 2>&1 | tee /root/nixos-infect.log diff --git a/nixos-infect b/nixos-infect index 3d24392..b04f95a 100755 --- a/nixos-infect +++ b/nixos-infect @@ -13,7 +13,7 @@ : "${STAGING_ACME:?STAGING_ACME variable is not set}" : "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" : "${DB_PASSWORD:?DB_PASSWORD variable is not set}" -: "${USER_PASS:?USER_PASS variable is not set}" +: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" @@ -293,6 +293,12 @@ findESP() { } prepareEnv() { + if ! USER_PASS="$(base64 -d <<<"$ENCODED_PASSWORD")"; then + echo "Error decoding ENCODED_PASSWORD from Base64!" + exit 1 + fi + readonly USER_PASS + isEFI=0 [ -d /sys/firmware/efi ] && isEFI=1 From c31521e1ca47bcf00ab392b839de74e9330f20d2 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 18:01:57 +0400 Subject: [PATCH 114/124] do not pass DB_PASSWORD, but generate --- .drone.yml | 1 - nixos-infect | 15 +++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.drone.yml b/.drone.yml index 5b6a5d1..d65bba1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,7 +33,6 @@ steps: API_TOKEN="$USER_PASS" CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/0f886d76e93dd366db7c53a8f6b672702910b99b.tar.gz - DB_PASSWORD="$USER_PASS" DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN diff --git a/nixos-infect b/nixos-infect index b04f95a..56c48b3 100755 --- a/nixos-infect +++ b/nixos-infect @@ -12,7 +12,6 @@ : "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}" : "${STAGING_ACME:?STAGING_ACME variable is not set}" : "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" -: "${DB_PASSWORD:?DB_PASSWORD variable is not set}" : "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" @@ -60,13 +59,16 @@ EOF } genSecrets() { + local dbpass + dbpass="$(shuf --random-source=/dev/urandom -erz -n32 {A..Z} {a..z} {0..9} | tr -d '\n')" + cat << EOF { "api": { "token": "$API_TOKEN", "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"] }, - "databasePassword": "$DB_PASSWORD", + "databasePassword": "$dbpass", "dns": { "apiKey": "$DNS_PROVIDER_TOKEN" }, @@ -374,7 +376,8 @@ checkEnv() { req xzcat || { echo "ERROR: Missing xzcat"; return 1; } req awk || { echo "ERROR: Missing awk"; return 1; } req cut || req df || { echo "ERROR: Missing coreutils (cut, df)"; return 1; } - req mkpasswd || { echo "ERROR: Missing mkpasswd"; return 1; } + req mkpasswd || { echo "ERROR: Missing mkpasswd"; return 1; } + req shuf || { echo "ERROR: Missing shuf"; return 1; } } # Download and execute the nix installer script. @@ -464,11 +467,11 @@ infect() { /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage } -set -o pipefail +set -o errtrace set -o nounset -set -o errexit -set -o xtrace +set -o pipefail shopt -s inherit_errexit +trap 'echo ${LINENO}: "$BASH_COMMAND"; exit 1' ERR genNetworkingConf From 83fe55b86ec5c78fd4f1ea3605c5ec4ef82a24ef Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 18:08:03 +0400 Subject: [PATCH 115/124] nixos-infect: sort variables --- nixos-infect | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nixos-infect b/nixos-infect index 56c48b3..6003e11 100755 --- a/nixos-infect +++ b/nixos-infect @@ -5,17 +5,18 @@ # - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect : "${API_TOKEN:?API_TOKEN variable is not set}" -: "${DOMAIN:?DOMAIN variable is not set}" -: "${LUSER:?LUSER variable is not set}" -: "${HOSTNAME:?HOSTNAME variable is not set}" -: "${PROVIDER:?PROVIDER variable is not set}" -: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}" -: "${STAGING_ACME:?STAGING_ACME variable is not set}" -: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" -: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" -: "${NIX_VERSION:?NIX_VERSION variable is not set}" -: "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" +: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" +: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}" +: "${DOMAIN:?DOMAIN variable is not set}" +: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" +: "${HOSTNAME:?HOSTNAME variable is not set}" +: "${LUSER:?LUSER variable is not set}" +: "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" +: "${NIX_VERSION:?NIX_VERSION variable is not set}" +: "${PROVIDER:?PROVIDER variable is not set}" +: "${STAGING_ACME:?STAGING_ACME variable is not set}" + : "${SSH_AUTHORIZED_KEY:=}" readonly NL=$'\n' From 2f9903599da06ec6d2b2acdeecd859864a83d10f Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 20:04:01 +0400 Subject: [PATCH 116/124] proper ERR trap --- .drone.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d65bba1..66e110a 100644 --- a/.drone.yml +++ b/.drone.yml @@ -28,8 +28,10 @@ steps: #! /usr/bin/env bash set -o allexport + set -o errtrace set -o pipefail - set -o errexit + shopt -s inherit_errexit + trap 'echo \$LINENO: \"\$BASH_COMMAND\"; exit 1' ERR API_TOKEN="$USER_PASS" CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/0f886d76e93dd366db7c53a8f6b672702910b99b.tar.gz From 0ec4fc7d5bfb67623f023ecf28299301b60742e6 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Wed, 27 Dec 2023 20:53:02 +0400 Subject: [PATCH 117/124] generate volumes and location values (disk device name) --- .drone.yml | 19 +++++++++++++++++-- nixos-infect | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.drone.yml b/.drone.yml index 66e110a..d997c8f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -48,10 +48,16 @@ steps: STAGING_ACME=true curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \ - | bash 2>&1 | tee /root/nixos-infect.log + | bash 2>&1 | tee /root/infect.log EOF - cat infect.sh - - http -v --check-status --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 user_data=@infect.sh automount:=false location=fsn1 + - location=fsn1 + - set -o pipefail + - > + volume_id="$(http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" format=ext4 size:=10 location=$location name=10gb-by-drone-ci automount:=false | jq -re .volume.id)" + # TODO pass linux device volume_id name to infect.sh script + - > + http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 volumes:=[$volume_id] user_data=@infect.sh automount:=false location=$location - name: dns environment: @@ -214,6 +220,15 @@ steps: echo "$ZONE_ID" - > echo "$result" | jq '.[] | "\(.id)"' | while read id ; do echo "Deleting $id"; http -v --check-status --ignore-stdin DELETE https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$(echo $id | cut -f 2 -d '"') Authorization:"Bearer $CLOUDFLARE_TOKEN"; done + - location="fsn1" + - volume_name="10gb-by-drone-ci" + - > + volumes_json="$(http -d --print=HBb --ignore-stdin GET https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" name=$volume_name)" + # as of 2023-12-27 Hetzner ignores "name" query parameter (and others too)!! + - > + volume_id="$(jq <<<"$volumes_json" -re "(.volumes[] | select(.name == \"$volume_name\")).id")" + - > + http -d --print=HBb --ignore-stdin DELETE https://api.hetzner.cloud/v1/volumes/$volume_id Authorization:"Bearer $PASSWORD" when: status: - failure diff --git a/nixos-infect b/nixos-infect index 6003e11..a1254fe 100755 --- a/nixos-infect +++ b/nixos-infect @@ -32,10 +32,27 @@ genOptionalSsh() { EOF } +# TODO receive disk device from outside +determine2ndDisk() { + case "$PROVIDER" in + hetzner) + echo "/dev/sdb" + ;; + digitalocean) + echo "/dev/sda" + ;; + *) + return 1 + ;; + esac +} + # Merge original userdata.json with deployment specific fields and print result. genUserdata() { - local HASHED_PASSWORD userdata_infect release - HASHED_PASSWORD="$(mkpasswd -m sha-512 "$USER_PASS")" + local hashed_password diskDeviceName userdata_infect + hashed_password="$(mkpasswd -m sha-512 "$USER_PASS")" + diskDevice="$(determine2ndDisk)" + diskDeviceName="${diskDevice##/dev}" userdata_infect=$(cat << EOF { @@ -48,9 +65,33 @@ genUserdata() { "provider": "$PROVIDER" }, "domain": "$DOMAIN", - "hashedMasterPassword": "$HASHED_PASSWORD", + "hashedMasterPassword": "$hashed_password", "hostname": "$HOSTNAME", - "username": "$LUSER" + "username": "$LUSER", + "volumes": [ + { + "device": "$diskDevice", + "mountPoint": "/volumes/$diskDeviceName", + "fsType": "ext4" + } + ], + "modules": { + "bitwarden": { + "location": "$diskDeviceName" + }, + "gitea": { + "location": "$diskDeviceName" + }, + "nextcloud": { + "location": "$diskDeviceName" + }, + "pleroma": { + "location": "$diskDeviceName" + }, + "simple-nixos-mailserver": { + "location": "$diskDeviceName" + } + } } EOF ) @@ -471,6 +512,7 @@ infect() { set -o errtrace set -o nounset set -o pipefail +set -o xtrace shopt -s inherit_errexit trap 'echo ${LINENO}: "$BASH_COMMAND"; exit 1' ERR From eaafba6f65432ad1cc44856b86c3ce3ceb18b570 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 00:35:36 +0400 Subject: [PATCH 118/124] fix ERR trap in infect.sh --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index d997c8f..8940600 100644 --- a/.drone.yml +++ b/.drone.yml @@ -31,7 +31,7 @@ steps: set -o errtrace set -o pipefail shopt -s inherit_errexit - trap 'echo \$LINENO: \"\$BASH_COMMAND\"; exit 1' ERR + trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR API_TOKEN="$USER_PASS" CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/0f886d76e93dd366db7c53a8f6b672702910b99b.tar.gz From 18c88f39cf6167ce320d160b7e806ca46b878959 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 00:45:07 +0400 Subject: [PATCH 119/124] ci: slightly reformat basic test logging --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 8940600..d8045e9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -140,11 +140,11 @@ steps: - curl https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -sS --fail -o root.pem - dig api.$DOMAIN - > - retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS" + retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status -sS --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS" - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp - > curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"' - - echo 'File to test nextcloud upload' > file + - echo 'File contents to test nextcloud upload' > file - > curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -d @file -u "admin:$USER_PASS" - > From 91073a50a3657a75e65d94f1121f584996cf00e4 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 00:56:47 +0400 Subject: [PATCH 120/124] additional check for 2nd volume --- nixos-infect | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nixos-infect b/nixos-infect index a1254fe..7cae9eb 100755 --- a/nixos-infect +++ b/nixos-infect @@ -32,19 +32,22 @@ genOptionalSsh() { EOF } -# TODO receive disk device from outside +# TODO receive disk device path from outside determine2ndDisk() { + local guess rootFsDev case "$PROVIDER" in hetzner) - echo "/dev/sdb" + guess="/dev/sdb" ;; digitalocean) - echo "/dev/sda" + guess="/dev/sda" ;; *) return 1 ;; esac + rootFsDev="$(findmnt -no source -T /)" + [[ -b "$guess" && "$guess" != "$rootFsDev" ]] && echo "$guess" } # Merge original userdata.json with deployment specific fields and print result. From 374711f324adfb394cc2b823b9b7f2d18d4d431d Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 03:12:43 +0400 Subject: [PATCH 121/124] optional variables: NIXOS_CONFIG_ID and VOLUME_DEV_PATH --- nixos-infect | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/nixos-infect b/nixos-infect index 7cae9eb..eefcf68 100755 --- a/nixos-infect +++ b/nixos-infect @@ -4,6 +4,7 @@ # - https://github.com/elitak/nixos-infect # - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect +# mandatory variables : "${API_TOKEN:?API_TOKEN variable is not set}" : "${CONFIG_URL:?CONFIG_URL variable is not set}" : "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}" @@ -12,12 +13,14 @@ : "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}" : "${HOSTNAME:?HOSTNAME variable is not set}" : "${LUSER:?LUSER variable is not set}" -: "${NIXOS_CONFIG_ID:?NIXOS_CONFIG_ID variable is not set}" : "${NIX_VERSION:?NIX_VERSION variable is not set}" : "${PROVIDER:?PROVIDER variable is not set}" : "${STAGING_ACME:?STAGING_ACME variable is not set}" +# optional variables +: "${NIXOS_CONFIG_ID:=default}" : "${SSH_AUTHORIZED_KEY:=}" +: "${VOLUME_DEV_PATH:=}" readonly NL=$'\n' readonly LOCAL_FLAKE_DIR="/etc/nixos" @@ -34,20 +37,24 @@ EOF # TODO receive disk device path from outside determine2ndDisk() { - local guess rootFsDev - case "$PROVIDER" in - hetzner) - guess="/dev/sdb" - ;; - digitalocean) - guess="/dev/sda" - ;; - *) - return 1 - ;; - esac + local volDevPath rootFsDev + if [ -n "$VOLUME_DEV_PATH" ]; then + volDevPath="$VOLUME_DEV_PATH" + else + case "$PROVIDER" in + hetzner) + volDevPath="/dev/sdb" + ;; + digitalocean) + volDevPath="/dev/sda" + ;; + *) + return 1 + ;; + esac + fi rootFsDev="$(findmnt -no source -T /)" - [[ -b "$guess" && "$guess" != "$rootFsDev" ]] && echo "$guess" + [[ -b "$volDevPath" && "$volDevPath" != "$rootFsDev" ]] && echo "$volDevPath" } # Merge original userdata.json with deployment specific fields and print result. From 36eb48ab3caa909e9a7ce487c402af607cea35e7 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 04:32:31 +0400 Subject: [PATCH 122/124] nix flake lock /etc/nixos --override-input sp-modules path:./sp-modules --- .drone.yml | 2 +- nixos-infect | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index d8045e9..4127996 100644 --- a/.drone.yml +++ b/.drone.yml @@ -34,7 +34,7 @@ steps: trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/0f886d76e93dd366db7c53a8f6b672702910b99b.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/9343332f1206995957f5c8e61ee2d7eafbbe6131.tar.gz DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN diff --git a/nixos-infect b/nixos-infect index eefcf68..a4f6fc5 100755 --- a/nixos-infect +++ b/nixos-infect @@ -480,7 +480,9 @@ installNix() { infect() { # this is needed solely for accepting the sp-module sub-flake # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478 - $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} --update-input sp-modules + cd /etc/nixos || return 1 + $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} \ + --override-input sp-modules path:./sp-modules echo "nix build the configuration flake..." if ! $NIX "${NIX_OPTS[@]}" build \ From 7cf0a1432db3baff2f68bc43b266d6a0aee8e697 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 13:23:08 +0400 Subject: [PATCH 123/124] update: working system.autoUpgrade --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 4127996..57ec8ec 100644 --- a/.drone.yml +++ b/.drone.yml @@ -34,7 +34,7 @@ steps: trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/9343332f1206995957f5c8e61ee2d7eafbbe6131.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/6d0239ad5c61c4bc708238908fe6814d931d15e7.tar.gz DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN From 6ece80adc246baed4a172d2cb08f1d3ef5c6357b Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Thu, 28 Dec 2023 13:30:25 +0400 Subject: [PATCH 124/124] update: system.autoUpgrade: no verbosity of nixos-rebuild --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index 57ec8ec..b494dea 100644 --- a/.drone.yml +++ b/.drone.yml @@ -34,7 +34,7 @@ steps: trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR API_TOKEN="$USER_PASS" - CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/6d0239ad5c61c4bc708238908fe6814d931d15e7.tar.gz + CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/4d18c8dba9765bde9040ddf1c285bec96ba23f43.tar.gz DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN DNS_PROVIDER_TYPE=CLOUDFLARE DOMAIN=$DOMAIN