SelfPrivacy
/
selfprivacy.org.app
10
7
Fork 5
Code Issues 75 Pull Requests 2 Projects 1 Releases 24 Wiki Activity

Better CI

pull/114/head
Alya Sirko 2022-09-14 16:21:46 +03:00
parent 81c97e5249
commit 96a7e9bf06
3 changed files with 94 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
.drone.yml
View File

@ -7,9 +7,17 @@ steps:
commands: commands:
- ln -s /var/lib/drone-runner-exec/.local $HOME/.local - ln -s /var/lib/drone-runner-exec/.local $HOME/.local
- name: Build - name: Run Tests
commands: commands:
- ./ci.py --run-ci-build - ./ci.py --ci-run-tests
- name: Build Linux Target
commands:
- ./ci.py --ci-build-linux
- name: Build APK Target
commands:
- ./ci.py --ci-build-apk
trigger: trigger:
event: event:
@ -29,8 +37,9 @@ steps:
- name: Prepare - name: Prepare
commands: commands:
- ln -s /var/lib/drone-runner-exec/.local $HOME/.local - ln -s /var/lib/drone-runner-exec/.local $HOME/.local
- if podman volume exists src; then podman volume rm -f src; podman volume create src; else podman volume create src; fi - if podman volume exists release; then podman volume rm -f release; podman volume create release; else podman volume create release; fi
- git archive --format=tar HEAD | podman volume import src - - git config user.email "builder@selfprivacy.org"
- git config user.name "Builder"
- name: Build Intermediate Linux Release Artifact (Binary) - name: Build Intermediate Linux Release Artifact (Binary)
commands: commands:
@ -80,6 +89,12 @@ steps:
commands: commands:
- ./ci.py --package-linux-archive - ./ci.py --package-linux-archive
- name: Push Artifacts to the Release Volume
commands:
- git add -v *.AppImage *.flatpak *.apk *.apk.idsig *.tar.zstd
- git commit -m Release
- git archive --format=tar HEAD | podman volume import release -
trigger: trigger:
event: event:
- tag - tag
@ -97,18 +112,18 @@ steps:
- name: Prepare - name: Prepare
commands: commands:
- ln -s /var/lib/drone-runner-exec/.local $HOME/.local - ln -s /var/lib/drone-runner-exec/.local $HOME/.local
- podman unshare podman volume mount src - podman unshare podman volume mount release
- name: Create Release and Deploy Artifacts - name: Create Release and Deploy Artifacts
commands: commands:
- podman unshare ./ci.py --deploy-gitea-release - ./ci.py --deploy-gitea-release
environment: environment:
GITEA_RELEASE_TOKEN: GITEA_RELEASE_TOKEN:
from_secret: GITEA_RELEASE_TOKEN from_secret: GITEA_RELEASE_TOKEN
- name: Deploy F-Droid Repo - name: Deploy F-Droid Repo
commands: commands:
- podman unshare ./ci.py --deploy-fdroid-repo - ./ci.py --deploy-fdroid-repo
environment: environment:
SSH_PRIVATE_KEY: SSH_PRIVATE_KEY:
from_secret: SSH_PRIVATE_KEY from_secret: SSH_PRIVATE_KEY

64
Dockerfile
View File

@ -1,48 +1,62 @@
FROM ubuntu:22.04 FROM ubuntu:22.04
ENV PACKAGES "build-essential openjdk-11-jdk-headless clang cmake curl git jq libblkid1 libblkid-dev libc6 libc6-dev libc-bin libcrypt1 libdbus-1-3 libexpat1 libffi7 libgcc-s1 libgcrypt20 libgcrypt20-dev libglib2.0-0 libglib2.0-dev libglu1-mesa libgpg-error0 libgtk-3-0 libgtk-3-dev liblz4-1 liblz4-dev liblzma5 liblzma-dev libmount1 libpcre3 libselinux1 libsepol2 libstdc++-10-dev libstdc++6 libuuid1 ninja-build pkg-config rsync unzip xz-utils zlib1g unzip libsecret-1-dev libsecret-tools libsecret-1-0 libjsoncpp-dev fuse flatpak-builder binutils coreutils desktop-file-utils fakeroot fuse libgdk-pixbuf2.0-dev patchelf python3-pip python3-setuptools squashfs-tools strace util-linux zsync" ARG PACKAGES="build-essential openjdk-11-jdk-headless clang cmake curl git jq libblkid1 \
ENV ANDROID_SDK_TOOLS_VERSION "commandlinetools-linux-8512546_latest" libblkid-dev libc6 libc6-dev libc-bin libcrypt1 libdbus-1-3 libexpat1 libffi7 \
ENV ANDROID_SDK_TOOLS_URL "https://dl.google.com/android/repository/commandlinetools-linux-8512546_latest.zip" libgcc-s1 libgcrypt20 libgcrypt20-dev libglib2.0-0 libglib2.0-dev libglu1-mesa \
ENV FLUTTER_VERSION "flutter_linux_3.3.1-stable" libgpg-error0 libgtk-3-0 libgtk-3-dev liblz4-1 liblz4-dev liblzma5 liblzma-dev \
ENV FLUTTER_URL "https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_3.3.1-stable.tar.xz" libmount1 libpcre3 libselinux1 libsepol2 libstdc++-10-dev libstdc++6 libuuid1 \
ENV FREEDESKTOP_SDK_VERSION "22.08" ninja-build pkg-config rsync unzip xz-utils zlib1g unzip libsecret-1-dev libsecret-tools \
libsecret-1-0 libjsoncpp-dev fuse flatpak-builder binutils coreutils desktop-file-utils \
fakeroot fuse libgdk-pixbuf2.0-dev patchelf python3-pip python3-setuptools squashfs-tools \
strace util-linux zsync"
ARG ANDROID_SDK_TOOLS_VERSION="8512546"
ARG ANDROID_SDK_TOOLS_URL="https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip"
ARG FLUTTER_VERSION="3.3.1"
ARG FLUTTER_URL="https://storage.googleapis.com/flutter_infra_release/releases/stable/linux/flutter_linux_${FLUTTER_VERSION}-stable.tar.xz"
ARG FREEDESKTOP_SDK_VERSION="22.08"
# Update packages # Update packages
RUN apt-get update && apt-get upgrade -y && apt-get install -y $PACKAGES RUN apt-get update && apt-get upgrade -y && apt-get install -y $PACKAGES
# Add a non-privileged user WORKDIR /opt
RUN useradd -d /var/lib/builder -m -r -s /bin/bash builder
USER builder
WORKDIR /var/lib/builder
# Install Android SDK # Install Android SDK
ADD --chown=builder $ANDROID_SDK_TOOLS_URL . ADD $ANDROID_SDK_TOOLS_URL .
RUN mkdir -p android-sdk/cmdline-tools && unzip $ANDROID_SDK_TOOLS_VERSION.zip \ RUN mkdir -p android-sdk/cmdline-tools && unzip commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip \
&& rm $ANDROID_SDK_TOOLS_VERSION.zip && mv cmdline-tools android-sdk/cmdline-tools/latest && rm commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip && mv cmdline-tools android-sdk/cmdline-tools/latest
# Install Flutter # Install Flutter
ADD --chown=builder $FLUTTER_URL . ADD $FLUTTER_URL .
RUN tar -vxf $FLUTTER_VERSION.tar.xz && rm $FLUTTER_VERSION.tar.xz RUN tar -vxf flutter_linux_${FLUTTER_VERSION}-stable.tar.xz && \
rm flutter_linux_${FLUTTER_VERSION}-stable.tar.xz
# Flutter doesn't work without write permissions, so fuck it, fuck.
RUN chmod -R 777 /opt/flutter
RUN git config --system --add safe.directory /opt/flutter
ENV ANDROID_HOME "/var/lib/builder/android-sdk" ENV ANDROID_HOME "/opt/android-sdk"
ENV ANDROID_SDK_ROOT "/var/lib/builder/android-sdk" ENV ANDROID_SDK_ROOT "${ANDROID_HOME}"
ENV PATH "$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:/var/lib/builder/flutter/bin:/var/lib/builder/.local/bin" ENV PATH "$PATH:$ANDROID_SDK_ROOT/cmdline-tools/latest/bin:/opt/flutter/bin"
# Install needed Android SDK packages # Install needed Android SDK packages
RUN yes | sdkmanager 'build-tools;30.0.3' 'platforms;android-29' 'platforms;android-30' 'platforms;android-31' RUN yes | sdkmanager 'build-tools;30.0.3' 'platforms;android-29' 'platforms;android-30' 'platforms;android-31'
WORKDIR /tmp
# Prepare dependencies for offline build # Prepare dependencies for offline build
RUN git clone --depth=1 --single-branch https://git.selfprivacy.org/kherel/selfprivacy.org.app.git deps RUN git clone --depth=1 --single-branch https://git.selfprivacy.org/kherel/selfprivacy.org.app.git deps
WORKDIR /var/lib/builder/deps WORKDIR /tmp/deps
RUN flutter build linux RUN flutter build linux
RUN flutter build apk RUN flutter build apk
WORKDIR /var/lib/builder WORKDIR /tmp
RUN rm -rf deps RUN rm -rf deps
# Install AppImage Builder # Install Python dependencies
RUN pip3 install --user appimage-builder RUN pip3 install appimage-builder bandit setuptools portalocker pytz pytest pytest-mock \
pytest-datadir huey gevent mnemonic coverage pylint pydantic \
typing-extensions psutil black fastapi uvicorn strawberry-graphql \
python-multipart python-dateutil pygments poetry graphql-core
# Install Flatpak dependencies # Install Flatpak dependencies
RUN flatpak --user remote-add flathub https://flathub.org/repo/flathub.flatpakrepo \ RUN flatpak remote-add flathub https://flathub.org/repo/flathub.flatpakrepo \
&& flatpak --user install -y org.freedesktop.Sdk/x86_64/$FREEDESKTOP_SDK_VERSION \ && flatpak install -y org.freedesktop.Sdk/x86_64/${FREEDESKTOP_SDK_VERSION} \
org.freedesktop.Platform/x86_64/$FREEDESKTOP_SDK_VERSION org.freedesktop.Platform/x86_64/${FREEDESKTOP_SDK_VERSION}

59
ci.py
View File

@ -1,43 +1,39 @@
#!/usr/bin/env python3 \#!/usr/bin/env python3
import os import os
import subprocess import subprocess
import yaml import yaml
import argparse import argparse
CONTAINER_IMAGE = "localhost/flutter-build-env" CONTAINER_IMAGE = "docker.io/alyasirko/flutter-build-env"
HOST_HOME = "/var/lib/drone-runner-exec" HOST_HOME = "/var/lib/drone-runner-exec"
CONTAINER_HOME = "/var/lib/builder" CONTAINER_HOME = "/tmp/builder"
APP_NAME = "pro.kherel.selfprivacy" APP_NAME = "pro.kherel.selfprivacy"
APP_VERSION_FULL = yaml.safe_load(open("pubspec.yaml", "r"))['version'] APP_VERSION_FULL = yaml.safe_load(open("pubspec.yaml", "r"))['version']
APP_SEMVER = APP_VERSION_FULL[:APP_VERSION_FULL.find("+")] APP_SEMVER = APP_VERSION_FULL[:APP_VERSION_FULL.find("+")]
APP_BUILD_ID = APP_VERSION_FULL[APP_VERSION_FULL.find("+"):][1::] APP_BUILD_ID = APP_VERSION_FULL[APP_VERSION_FULL.find("+"):][1::]
HOST_MOUNTED_VOLUME = f"{HOST_HOME}/.local/share/containers/storage/volumes/src/_data" HOST_MOUNTED_VOLUME = f"{HOST_HOME}/.local/share/containers/storage/volumes/release/_data"
# Environments # Environments
def podman_offline(dir, *args): def podman_offline(dir, *args):
subprocess.run(["podman", "run", "--rm", "--network=none", f"--workdir={dir}", subprocess.run(["podman", "run", "--rm", "--network=none", f"--workdir={dir}",
"-v", f"src:{CONTAINER_HOME}/src:U", "-v", os.getcwd() + f":{CONTAINER_HOME}/src",
"-v", f"{HOST_HOME}/fdroid:{CONTAINER_HOME}/fdroid:U", "-v", f"{HOST_HOME}/fdroid:{CONTAINER_HOME}/fdroid",
"-v", f"{HOST_HOME}/fdroid-keystore:{CONTAINER_HOME}/fdroid/fdroid-keystore:U", "-v", f"{HOST_HOME}/fdroid-keystore:{CONTAINER_HOME}/fdroid/fdroid-keystore",
"-v", f"{HOST_HOME}/standalone-keystore:{CONTAINER_HOME}/fdroid/standalone-keystore:U", "-v", f"{HOST_HOME}/standalone-keystore:{CONTAINER_HOME}/fdroid/standalone-keystore",
"--env", "FDROID_KEYSTORE_PASS=" + os.environ.get('FDROID_KEYSTORE_PASS'), "--env", "FDROID_KEYSTORE_PASS=" + os.environ.get('FDROID_KEYSTORE_PASS'),
"--env", "STANDALONE_KEYSTORE_PASS=" + os.environ.get('STANDALONE_KEYSTORE_PASS'), "--env", "STANDALONE_KEYSTORE_PASS=" + os.environ.get('STANDALONE_KEYSTORE_PASS'),
"--user", os.getuid() + ":" + os.getgid(), "--userns=keep-id",
CONTAINER_IMAGE, "bash", "-c", ' '.join(args) CONTAINER_IMAGE, "bash", "-c", ' '.join(args)
]) ])
def podman_online(dir, *args): def podman_online(dir, *args):
subprocess.run(["podman", "run", "--rm", "--privileged", f"--workdir={dir}", subprocess.run(["podman", "run", "--rm", "--privileged", f"--workdir={dir}",
"-v", f"src:{CONTAINER_HOME}/src:U", "-v", os.getcwd() + f":{CONTAINER_HOME}/src",
CONTAINER_IMAGE, "bash", "-c", ' '.join(args) "--user", os.getuid() + ":" + os.getgid(), "--userns=keep-id",
])
def podman_ci(dir, *args):
subprocess.run(["podman", "run", "--rm", "--privileged", f"--workdir={dir}",
"-v", os.getcwd() + f":{CONTAINER_HOME}/src:U",
CONTAINER_IMAGE, "bash", "-c", ' '.join(args) CONTAINER_IMAGE, "bash", "-c", ' '.join(args)
]) ])
@ -52,11 +48,11 @@ def build_apk():
podman_offline(f"{CONTAINER_HOME}/src", "flutter build apk") podman_offline(f"{CONTAINER_HOME}/src", "flutter build apk")
def sign_apk_standalone(): def sign_apk_standalone():
podman_offline(f"{CONTAINER_HOME}/fdroid", podman_offline(f"{CONTAINER_HOME}/src",
"zipalign -f -v 4 ../src/build/app/outputs/flutter-apk/app-release.apk", "zipalign -f -v 4 build/app/outputs/flutter-apk/app-release.apk",
f"standalone_{APP_NAME}-{APP_SEMVER}.apk") f"standalone_{APP_NAME}-{APP_SEMVER}.apk")
podman_offline(f"{CONTAINER_HOME}/fdroid", podman_offline(f"{CONTAINER_HOME}/fdroid",
"apksigner sign --ks standalone-keystore --ks-key-alias standalone --ks-pass", "apksigner sign --ks ../fdroid/standalone-keystore --ks-key-alias standalone --ks-pass",
f"env:STANDALONE_KEYSTORE_PASS standalone_{APP_NAME}-{APP_SEMVER}.apk") f"env:STANDALONE_KEYSTORE_PASS standalone_{APP_NAME}-{APP_SEMVER}.apk")
def sign_apk_fdroid(): def sign_apk_fdroid():
@ -83,8 +79,8 @@ def deploy_gitea_release():
"--url", "https://git.selfprivacy.org"]) "--url", "https://git.selfprivacy.org"])
subprocess.run(["tea", "releases", "create", "--repo", os.environ.get('DRONE_REPO'), subprocess.run(["tea", "releases", "create", "--repo", os.environ.get('DRONE_REPO'),
"--tag", os.environ.get('DRONE_SEMVER'), "--title", os.environ.get('DRONE_SEMVER'), "--tag", os.environ.get('DRONE_SEMVER'), "--title", os.environ.get('DRONE_SEMVER'),
"--asset", f"{HOST_HOME}/fdroid/standalone_{APP_NAME}-{APP_SEMVER}.apk", "--asset", f"{HOST_MOUNTED_VOLUME}/standalone_{APP_NAME}-{APP_SEMVER}.apk",
"--asset", f"{HOST_HOME}/fdroid/standalone_{APP_NAME}-{APP_SEMVER}.apk.idsig", "--asset", f"{HOST_MOUNTED_VOLUME}/standalone_{APP_NAME}-{APP_SEMVER}.apk.idsig",
"--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage", "--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage",
"--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage.zsync", "--asset", f"{HOST_MOUNTED_VOLUME}/SelfPrivacy-{APP_SEMVER}-x86_64.AppImage.zsync",
"--asset", f"{HOST_MOUNTED_VOLUME}/{APP_NAME}-{APP_SEMVER}.flatpak", "--asset", f"{HOST_MOUNTED_VOLUME}/{APP_NAME}-{APP_SEMVER}.flatpak",
@ -96,9 +92,14 @@ def deploy_fdroid_repo():
scp -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r {HOST_HOME}/fdroid/repo/* deployer@selfprivacy.org:/var/www/fdroid.selfprivacy.org scp -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -r {HOST_HOME}/fdroid/repo/* deployer@selfprivacy.org:/var/www/fdroid.selfprivacy.org
"""], shell=True) """], shell=True)
def run_ci_build(): def ci_build_linux():
podman_ci(f"{CONTAINER_HOME}/src", "flutter build linux --debug") podman_online(f"{CONTAINER_HOME}/src", "flutter build linux --debug")
podman_ci(f"{CONTAINER_HOME}/src", "flutter build apk --debug")
def ci_build_apk():
podman_online(f"{CONTAINER_HOME}/src", "flutter build apk --debug")
def ci_run_tests():
podman_online(f"{CONTAINER_HOME}/src", "flutter test")
# Arguments # Arguments
@ -114,7 +115,9 @@ if __name__ == "__main__":
group.add_argument("--package-linux-archive", action="store_true") group.add_argument("--package-linux-archive", action="store_true")
group.add_argument("--deploy-gitea-release", action="store_true", help="depends on $GITEA_RELEASE_TOKEN") group.add_argument("--deploy-gitea-release", action="store_true", help="depends on $GITEA_RELEASE_TOKEN")
group.add_argument("--deploy-fdroid-repo", action="store_true", help="depends on $SSH_PRIVATE_KEY") group.add_argument("--deploy-fdroid-repo", action="store_true", help="depends on $SSH_PRIVATE_KEY")
group.add_argument("--run-ci-build", action="store_true") group.add_argument("--ci-build-linux", action="store_true")
group.add_argument("--ci-build-apk", action="store_true")
group.add_argument("--ci-run-tests", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if args.build_linux: if args.build_linux:
@ -135,5 +138,9 @@ elif args.deploy_gitea_release:
deploy_gitea_release() deploy_gitea_release()
elif args.deploy_fdroid_repo: elif args.deploy_fdroid_repo:
deploy_fdroid_repo() deploy_fdroid_repo()
elif args.run_ci_build: elif args.ci_build_linux:
run_ci_build() ci_build_linux()
elif args.ci_build_apk:
ci_build_apk()
elif args.ci_run_tests:
ci_run_tests()