From 96a7e9bf06c9c7b36701e2c18c02ad926270a22f Mon Sep 17 00:00:00 2001 From: Alya Sirko Date: Wed, 14 Sep 2022 16:21:46 +0300 Subject: [PATCH] Better CI --- .drone.yml | 29 +++++++++++++++++++------ Dockerfile | 64 +++++++++++++++++++++++++++++++++--------------------- ci.py | 59 +++++++++++++++++++++++++++---------------------- 3 files changed, 94 insertions(+), 58 deletions(-) diff --git a/.drone.yml b/.drone.yml index 13ec2c42..2201e7df 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,9 +7,17 @@ steps: commands: - ln -s /var/lib/drone-runner-exec/.local $HOME/.local - - name: Build + - name: Run Tests commands: - - ./ci.py --run-ci-build + - ./ci.py --ci-run-tests + + - name: Build Linux Target + commands: + - ./ci.py --ci-build-linux + + - name: Build APK Target + commands: + - ./ci.py --ci-build-apk trigger: event: @@ -29,8 +37,9 @@ steps: - name: Prepare commands: - ln -s /var/lib/drone-runner-exec/.local $HOME/.local - - if podman volume exists src; then podman volume rm -f src; podman volume create src; else podman volume create src; fi - - git archive --format=tar HEAD | podman volume import src - + - if podman volume exists release; then podman volume rm -f release; podman volume create release; else podman volume create release; fi + - git config user.email "builder@selfprivacy.org" + - git config user.name "Builder" - name: Build Intermediate Linux Release Artifact (Binary) commands: @@ -80,6 +89,12 @@ steps: commands: - ./ci.py --package-linux-archive + - name: Push Artifacts to the Release Volume + commands: + - git add -v *.AppImage *.flatpak *.apk *.apk.idsig *.tar.zstd + - git commit -m Release + - git archive --format=tar HEAD | podman volume import release - + trigger: event: - tag @@ -97,18 +112,18 @@ steps: - name: Prepare commands: - ln -s /var/lib/drone-runner-exec/.local $HOME/.local - - podman unshare podman volume mount src + - podman unshare podman volume mount release - name: Create Release and Deploy Artifacts commands: - - podman unshare ./ci.py --deploy-gitea-release + - ./ci.py --deploy-gitea-release environment: GITEA_RELEASE_TOKEN: from_secret: GITEA_RELEASE_TOKEN - name: Deploy F-Droid Repo commands: - - podman unshare ./ci.py --deploy-fdroid-repo + - ./ci.py --deploy-fdroid-repo environment: SSH_PRIVATE_KEY: from_secret: SSH_PRIVATE_KEY diff --git a/Dockerfile b/Dockerfile index 7efdf1ed..ef439918 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,62 @@ FROM ubuntu:22.04 -ENV PACKAGES "build-essential openjdk-11-jdk-headless clang cmake curl git jq libblkid1 libblkid-dev libc6 libc6-dev libc-bin libcrypt1 libdbus-1-3 libexpat1 libffi7 libgcc-s1 libgcrypt20 libgcrypt20-dev libglib2.0-0 libglib2.0-dev libglu1-mesa libgpg-error0 libgtk-3-0 libgtk-3-dev liblz4-1 liblz4-dev liblzma5 liblzma-dev libmount1 libpcre3 libselinux1 libsepol2 libstdc++-10-dev libstdc++6 libuuid1 ninja-build pkg-config rsync unzip xz-utils zlib1g unzip libsecret-1-dev libsecret-tools libsecret-1-0 libjsoncpp-dev fuse flatpak-builder binutils coreutils desktop-file-utils fakeroot fuse libgdk-pixbuf2.0-dev patchelf python3-pip python3-setuptools squashfs-tools strace util-linux zsync" -ENV ANDROID_SDK_TOOLS_VERSION "commandlinetools-linux-8512546_latest" -ENV ANDROID_SDK_TOOLS_URL "https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip" -ENV FLUTTER_VERSION "flutter_linux_3.3.1-stable" -ENV FLUTTER_URL "https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_3.3.1-stable.tar.xz" -ENV FREEDESKTOP_SDK_VERSION "22.08" +ARG PACKAGES="build-essential openjdk-11-jdk-headless clang cmake curl git jq libblkid1 \ + libblkid-dev libc6 libc6-dev libc-bin libcrypt1 libdbus-1-3 libexpat1 libffi7 \ + libgcc-s1 libgcrypt20 libgcrypt20-dev libglib2.0-0 libglib2.0-dev libglu1-mesa \ + libgpg-error0 libgtk-3-0 libgtk-3-dev liblz4-1 liblz4-dev liblzma5 liblzma-dev \ + libmount1 libpcre3 libselinux1 libsepol2 libstdc++-10-dev libstdc++6 libuuid1 \ + ninja-build pkg-config rsync unzip xz-utils zlib1g unzip libsecret-1-dev libsecret-tools \ + libsecret-1-0 libjsoncpp-dev fuse flatpak-builder binutils coreutils desktop-file-utils \ + fakeroot fuse libgdk-pixbuf2.0-dev patchelf python3-pip python3-setuptools squashfs-tools \ + strace util-linux zsync" +ARG ANDROID_SDK_TOOLS_VERSION="8512546" +ARG ANDROID_SDK_TOOLS_URL="https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip" +ARG FLUTTER_VERSION="3.3.1" +ARG FLUTTER_URL="https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_${FLUTTER_VERSION}-stable.tar.xz" +ARG FREEDESKTOP_SDK_VERSION="22.08" # Update packages RUN apt-get update && apt-get upgrade -y && apt-get install -y $PACKAGES -# Add a non-privileged user -RUN useradd -d /var/lib/builder -m -r -s /bin/bash builder -USER builder -WORKDIR /var/lib/builder +WORKDIR /opt # Install Android SDK -ADD --chown=builder $ANDROID_SDK_TOOLS_URL . -RUN mkdir -p android-sdk/cmdline-tools && unzip $ANDROID_SDK_TOOLS_VERSION.zip \ - && rm $ANDROID_SDK_TOOLS_VERSION.zip && mv cmdline-tools android-sdk/cmdline-tools/latest +ADD $ANDROID_SDK_TOOLS_URL . +RUN mkdir -p android-sdk/cmdline-tools && unzip commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip \ + && rm commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip && mv cmdline-tools android-sdk/cmdline-tools/latest # Install Flutter -ADD --chown=builder $FLUTTER_URL . -RUN tar -vxf $FLUTTER_VERSION.tar.xz && rm $FLUTTER_VERSION.tar.xz +ADD $FLUTTER_URL . +RUN tar -vxf flutter_linux_${FLUTTER_VERSION}-stable.tar.xz && \ + rm flutter_linux_${FLUTTER_VERSION}-stable.tar.xz +# Flutter doesn't work without write permissions, so fuck it, fuck. +RUN chmod -R 777 /opt/flutter +RUN git config --system --add safe.directory /opt/flutter -ENV ANDROID_HOME "/var/lib/builder/android-sdk" -ENV ANDROID_SDK_ROOT "/var/lib/builder/android-sdk" -ENV PATH "$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:/var/lib/builder/flutter/bin:/var/lib/builder/.local/bin" +ENV ANDROID_HOME "/opt/android-sdk" +ENV ANDROID_SDK_ROOT "${ANDROID_HOME}" +ENV PATH "$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:/opt/flutter/bin" # Install needed Android SDK packages RUN yes | sdkmanager 'build-tools;30.0.3' 'platforms;android-29' 'platforms;android-30' 'platforms;android-31' +WORKDIR /tmp + # Prepare dependencies for offline build RUN git clone --depth=1 --single-branch https://git.selfprivacy.org/kherel/selfprivacy.org.app.git deps -WORKDIR /var/lib/builder/deps +WORKDIR /tmp/deps RUN flutter build linux RUN flutter build apk -WORKDIR /var/lib/builder +WORKDIR /tmp RUN rm -rf deps -# Install AppImage Builder -RUN pip3 install --user appimage-builder +# Install Python dependencies +RUN pip3 install appimage-builder bandit setuptools portalocker pytz pytest pytest-mock \ + pytest-datadir huey gevent mnemonic coverage pylint pydantic \ + typing-extensions psutil black fastapi uvicorn strawberry-graphql \ + python-multipart python-dateutil pygments poetry graphql-core # Install Flatpak dependencies -RUN flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo \ - && flatpak --user install -y org.freedesktop.Sdk/x86_64/$FREEDESKTOP_SDK_VERSION \ - org.freedesktop.Platform/x86_64/$FREEDESKTOP_SDK_VERSION +RUN flatpak remote-add flathub https://flathub.org/repo/flathub.flatpakrepo \ + && flatpak install -y org.freedesktop.Sdk/x86_64/${FREEDESKTOP_SDK_VERSION} \ + org.freedesktop.Platform/x86_64/${FREEDESKTOP_SDK_VERSION} diff --git a/ci.py b/ci.py index d8412580..b0b3a158 100755 --- a/ci.py +++ b/ci.py @@ -1,43 +1,39 @@ -#!/usr/bin/env python3 +\#!/usr/bin/env python3 import os import subprocess import yaml import argparse -CONTAINER_IMAGE = "localhost/flutter-build-env" +CONTAINER_IMAGE = "docker.io/alyasirko/flutter-build-env" HOST_HOME = "/var/lib/drone-runner-exec" -CONTAINER_HOME = "/var/lib/builder" +CONTAINER_HOME = "/tmp/builder" APP_NAME = "pro.kherel.selfprivacy" APP_VERSION_FULL = yaml.safe_load(open("pubspec.yaml", "r"))['version'] APP_SEMVER = APP_VERSION_FULL[:APP_VERSION_FULL.find("+")] APP_BUILD_ID = APP_VERSION_FULL[APP_VERSION_FULL.find("+"):][1::] -HOST_MOUNTED_VOLUME = f"{HOST_HOME}/.local/share/containers/storage/volumes/src/_data" +HOST_MOUNTED_VOLUME = f"{HOST_HOME}/.local/share/containers/storage/volumes/release/_data" # Environments def podman_offline(dir, *args): subprocess.run(["podman", "run", "--rm", "--network=none", f"--workdir={dir}", - "-v", f"src:{CONTAINER_HOME}/src:U", - "-v", f"{HOST_HOME}/fdroid:{CONTAINER_HOME}/fdroid:U", - "-v", f"{HOST_HOME}/fdroid-keystore:{CONTAINER_HOME}/fdroid/fdroid-keystore:U", - "-v", f"{HOST_HOME}/standalone-keystore:{CONTAINER_HOME}/fdroid/standalone-keystore:U", + "-v", os.getcwd() + f":{CONTAINER_HOME}/src", + "-v", f"{HOST_HOME}/fdroid:{CONTAINER_HOME}/fdroid", + "-v", f"{HOST_HOME}/fdroid-keystore:{CONTAINER_HOME}/fdroid/fdroid-keystore", + "-v", f"{HOST_HOME}/standalone-keystore:{CONTAINER_HOME}/fdroid/standalone-keystore", "--env", "FDROID_KEYSTORE_PASS=" + os.environ.get('FDROID_KEYSTORE_PASS'), "--env", "STANDALONE_KEYSTORE_PASS=" + os.environ.get('STANDALONE_KEYSTORE_PASS'), + "--user", os.getuid() + ":" + os.getgid(), "--userns=keep-id", CONTAINER_IMAGE, "bash", "-c", ' '.join(args) ]) def podman_online(dir, *args): subprocess.run(["podman", "run", "--rm", "--privileged", f"--workdir={dir}", - "-v", f"src:{CONTAINER_HOME}/src:U", - CONTAINER_IMAGE, "bash", "-c", ' '.join(args) - ]) - -def podman_ci(dir, *args): - subprocess.run(["podman", "run", "--rm", "--privileged", f"--workdir={dir}", - "-v", os.getcwd() + f":{CONTAINER_HOME}/src:U", + "-v", os.getcwd() + f":{CONTAINER_HOME}/src", + "--user", os.getuid() + ":" + os.getgid(), "--userns=keep-id", CONTAINER_IMAGE, "bash", "-c", ' '.join(args) ]) @@ -52,11 +48,11 @@ def build_apk(): podman_offline(f"{CONTAINER_HOME}/src", "flutter build apk") def sign_apk_standalone(): - podman_offline(f"{CONTAINER_HOME}/fdroid", - "zipalign -f -v 4 ../src/build/app/outputs/flutter-apk/app-release.apk", + podman_offline(f"{CONTAINER_HOME}/src", + "zipalign -f -v 4 build/app/outputs/flutter-apk/app-release.apk", f"standalone_{APP_NAME}-{APP_SEMVER}.apk") podman_offline(f"{CONTAINER_HOME}/fdroid", - "apksigner sign --ks standalone-keystore --ks-key-alias standalone --ks-pass", + "apksigner sign --ks ../fdroid/standalone-keystore --ks-key-alias standalone --ks-pass", f"env:STANDALONE_KEYSTORE_PASS standalone_{APP_NAME}-{APP_SEMVER}.apk") def sign_apk_fdroid(): @@ -83,8 +79,8 @@ def deploy_gitea_release(): "--url", "https://git.selfprivacy.org"]) subprocess.run(["tea", "releases", "create", "--repo", os.environ.get('DRONE_REPO'), "--tag", os.environ.get('DRONE_SEMVER'), "--title", os.environ.get('DRONE_SEMVER'), - "--asset", f"{HOST_HOME}/fdroid/standalone_{APP_NAME}-{APP_SEMVER}.apk", - "--asset", f"{HOST_HOME}/fdroid/standalone_{APP_NAME}-{APP_SEMVER}.apk.idsig", + "--asset", f"{HOST_MOUNTED_VOLUME}/standalone_{APP_NAME}-{APP_SEMVER}.apk", + "--asset", f"{HOST_MOUNTED_VOLUME}/standalone_{APP_NAME}-{APP_SEMVER}.apk.idsig", "--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage", "--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage.zsync", "--asset", f"{HOST_MOUNTED_VOLUME}/{APP_NAME}-{APP_SEMVER}.flatpak", @@ -96,9 +92,14 @@ def deploy_fdroid_repo(): scp -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r {HOST_HOME}/fdroid/repo/* deployer@selfprivacy.org:/var/www/fdroid.selfprivacy.org """], shell=True) -def run_ci_build(): - podman_ci(f"{CONTAINER_HOME}/src", "flutter build linux --debug") - podman_ci(f"{CONTAINER_HOME}/src", "flutter build apk --debug") +def ci_build_linux(): + podman_online(f"{CONTAINER_HOME}/src", "flutter build linux --debug") + +def ci_build_apk(): + podman_online(f"{CONTAINER_HOME}/src", "flutter build apk --debug") + +def ci_run_tests(): + podman_online(f"{CONTAINER_HOME}/src", "flutter test") # Arguments @@ -114,7 +115,9 @@ if __name__ == "__main__": group.add_argument("--package-linux-archive", action="store_true") group.add_argument("--deploy-gitea-release", action="store_true", help="depends on $GITEA_RELEASE_TOKEN") group.add_argument("--deploy-fdroid-repo", action="store_true", help="depends on $SSH_PRIVATE_KEY") - group.add_argument("--run-ci-build", action="store_true") + group.add_argument("--ci-build-linux", action="store_true") + group.add_argument("--ci-build-apk", action="store_true") + group.add_argument("--ci-run-tests", action="store_true") args = parser.parse_args() if args.build_linux: @@ -135,5 +138,9 @@ elif args.deploy_gitea_release: deploy_gitea_release() elif args.deploy_fdroid_repo: deploy_fdroid_repo() -elif args.run_ci_build: - run_ci_build() +elif args.ci_build_linux: + ci_build_linux() +elif args.ci_build_apk: + ci_build_apk() +elif args.ci_run_tests: + ci_run_tests()