From ae170232c0b0dbfbc0c6d30d2b4adb3b445f462c Mon Sep 17 00:00:00 2001 From: Jose Quintana Date: Tue, 9 May 2023 22:07:11 +0200 Subject: [PATCH] refactor: simplify ci scripts --- .cirrus.yml | 2 +- .github/workflows/devel.docker.yml | 4 ++-- .github/workflows/devel.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- ci/cargo.sh | 63 --------------------------------------------------------------- ci/get_latest_release.sh | 18 ------------------ ci/install-tools.sh | 36 ------------------------------------ ci/upload_release_file.sh | 41 ----------------------------------------- scripts/ci/cargo.sh | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/ci/get_latest_release.sh | 18 ++++++++++++++++++ scripts/ci/install_tools.sh | 36 ++++++++++++++++++++++++++++++++++++ scripts/ci/upload_release_file.sh | 41 +++++++++++++++++++++++++++++++++++++++++ 12 files changed, 165 insertions(+), 165 deletions(-) delete mode 100755 ci/cargo.sh delete mode 100755 ci/get_latest_release.sh delete mode 100755 ci/install-tools.sh delete mode 100755 ci/upload_release_file.sh create mode 100755 scripts/ci/cargo.sh create mode 100755 scripts/ci/get_latest_release.sh create mode 100755 scripts/ci/install_tools.sh create mode 100755 scripts/ci/upload_release_file.sh diff --git a/.cirrus.yml b/.cirrus.yml index 3bf2bf7..a39a2d8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -75,4 +75,4 @@ task: cp {README.md,LICENSE-APACHE,LICENSE-MIT} "$staging/" tar czf "$staging.tar.gz" "$staging" upload_script: | - ./ci/upload_release_file.sh + ./scripts/ci/upload_release_file.sh diff --git a/.github/workflows/devel.docker.yml b/.github/workflows/devel.docker.yml index 191cb8b..88c5af9 100644 --- a/.github/workflows/devel.docker.yml +++ b/.github/workflows/devel.docker.yml @@ -8,7 +8,7 @@ on: - trying paths: - docker/**/Dockerfile - - ci/**/*.sh + - scripts/ci/**/*.sh env: DOCKER_IMAGE: localhost:5000/github.com/static-web-server/static-web-server @@ -78,7 +78,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - ci/get_latest_release.sh + scripts/ci/get_latest_release.sh cat /tmp/version >> $GITHUB_ENV cat /tmp/version echo "SERVER_DOCKERFILE=./docker/alpine/Dockerfile" >> $GITHUB_ENV diff --git a/.github/workflows/devel.yml b/.github/workflows/devel.yml index 1480b87..b10a5fa 100644 --- a/.github/workflows/devel.yml +++ b/.github/workflows/devel.yml @@ -144,7 +144,7 @@ jobs: - name: Install Linux/BSD tools if: ${{ !contains(matrix.os, 'windows') }} - run: ci/install-tools.sh --target=${{ matrix.target }} + run: scripts/ci/install_tools.sh --target=${{ matrix.target }} - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -157,7 +157,7 @@ jobs: run: | # Use 'cargo.sh' script for Android targets only if [[ "${{ matrix.build }}" == "linux-android-armv7" ]] || [[ "${{ matrix.build }}" == "linux-android-arm64" ]]; then - echo "CARGO_BIN=ci/cargo.sh" >> $GITHUB_ENV + echo "CARGO_BIN=scripts/ci/cargo.sh" >> $GITHUB_ENV else cargo install cross@^0.2 echo "CARGO_BIN=cross" >> $GITHUB_ENV diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f2b25cf..54c99e4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -146,7 +146,7 @@ jobs: - name: Install Linux/BSD tools if: ${{ !contains(matrix.os, 'windows') }} - run: ci/install-tools.sh --target=${{ matrix.target }} + run: scripts/ci/install_tools.sh --target=${{ matrix.target }} - name: Install Rust uses: dtolnay/rust-toolchain@stable @@ -160,7 +160,7 @@ jobs: if [ "${{ matrix.os }}" = "ubuntu-22.04" ]; then # Use 'cargo.sh' script for Android targets only if [[ "${{ matrix.build }}" == "linux-android-armv7" ]] || [[ "${{ matrix.build }}" == "linux-android-arm64" ]]; then - echo "CARGO_BIN=ci/cargo.sh" >> $GITHUB_ENV + echo "CARGO_BIN=scripts/ci/cargo.sh" >> $GITHUB_ENV else cargo install cross@^0.2 echo "CARGO_BIN=cross" >> $GITHUB_ENV diff --git a/ci/cargo.sh b/ci/cargo.sh deleted file mode 100755 index 23efb4e..0000000 --- a/ci/cargo.sh +++ /dev/null @@ -1,63 +0,0 @@ -# Adapted from https://github.com/briansmith/ring/blob/main/mk/cargo.sh - -set -eux -o pipefail - -OS_NAME=$(uname -s | tr A-Z a-z) - -# Avoid putting the Android tools in `$PATH` because there are tools in this -# directory like `clang` that would conflict with the same-named tools that may -# be needed to compile the build script, or to compile for other targets. -if [ -n "${ANDROID_HOME-}" ]; then - # Keep the next line in sync with the corresponding line in install-build-tools.sh. - ndk_version=25.2.9519653 - ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT:-${ANDROID_HOME}/ndk/$ndk_version} -fi -if [ -n "${ANDROID_NDK_ROOT-}" ]; then - android_tools=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/${OS_NAME}-x86_64/bin -fi - -for arg in $*; do - case $arg in - --target=*) - target=${arg#*=} - ;; - *) - ;; - esac -done - -case $target in - aarch64-linux-android) - export CC_aarch64_linux_android=$android_tools/aarch64-linux-android21-clang - export AR_aarch64_linux_android=$android_tools/llvm-ar - export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=$android_tools/aarch64-linux-android21-clang - ;; - # aarch64-unknown-linux-gnu) - # export CC_aarch64_unknown_linux_gnu=clang-$llvm_version - # export AR_aarch64_unknown_linux_gnu=llvm-ar-$llvm_version - # export CFLAGS_aarch64_unknown_linux_gnu="--sysroot=/usr/aarch64-linux-gnu" - # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc - # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="$qemu_aarch64" - # ;; - # aarch64-unknown-linux-musl) - # export CC_aarch64_unknown_linux_musl=clang-$llvm_version - # export AR_aarch64_unknown_linux_musl=llvm-ar-$llvm_version - # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="$rustflags_self_contained" - # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER="$qemu_aarch64" - # ;; - # arm-unknown-linux-gnueabihf) - # export CC_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc - # export AR_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc-ar - # export CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc - # export CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER="$qemu_arm" - # ;; - armv7-linux-androideabi) - export CC_armv7_linux_androideabi=$android_tools/armv7a-linux-androideabi19-clang - export AR_armv7_linux_androideabi=$android_tools/llvm-ar - export CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER=$android_tools/armv7a-linux-androideabi19-clang - ;; - *) - ;; -esac - -cargo "$@" diff --git a/ci/get_latest_release.sh b/ci/get_latest_release.sh deleted file mode 100755 index 022605b..0000000 --- a/ci/get_latest_release.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -echo "Getting latest release number..." - -curl \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/repos/static-web-server/static-web-server/releases?per_page=5&page=1" | \ -jq -c "[ .[] | select( .tag_name | contains(\"v2.\")) ] | .[0]" | \ -jq -r ".tag_name" > /tmp/version - -version=$(cat /tmp/version) - -echo "SERVER_VERSION=${version#*v}" > /tmp/version - -echo "Version saved on '/tmp/version'" diff --git a/ci/install-tools.sh b/ci/install-tools.sh deleted file mode 100755 index d1dee42..0000000 --- a/ci/install-tools.sh +++ /dev/null @@ -1,36 +0,0 @@ -# Adapted from https://github.com/briansmith/ring/blob/main/mk/install-build-tools.sh - -set -eux -o pipefail - -target=$1 - -case $target in ---target*android*) - # https://blog.rust-lang.org/2023/01/09/android-ndk-update-r25.html says - # "Going forward the Android platform will target the most recent LTS NDK, - # allowing Rust developers to access platform features sooner. These updates - # should occur yearly and will be announced in release notes." Assume that - # means that we should always prefer to be using the latest 25.x.y version of - # the NDK until the Rust project announces that we should use a higher major - # version number. - # - # TODO: This should probably be implemented as a map of Rust toolchain version - # to NDK version; e.g. our MSRV might (only) support an older NDK than the - # latest stable Rust toolchain. - # - # Keep the following line in sync with the corresponding line in cargo.sh. - ndk_version=25.2.9519653 - - mkdir -p "${ANDROID_HOME}/licenses" - android_license_file="${ANDROID_HOME}/licenses/android-sdk-license" - accept_android_license=24333f8a63b6825ea9c5514f83c2829b004d1fee - grep --quiet --no-messages "$accept_android_license" "$android_license_file" \ - || echo $accept_android_license >> "$android_license_file" - "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" - - # XXX: Older Rust toolchain versions link with `-lgcc` instead of `-lunwind`; - # see https://github.com/rust-lang/rust/pull/85806. - find -L ${ANDROID_NDK_ROOT:-${ANDROID_HOME}/ndk/$ndk_version} -name libunwind.a \ - -execdir sh -c 'echo "INPUT(-lunwind)" > libgcc.a' \; - ;; -esac diff --git a/ci/upload_release_file.sh b/ci/upload_release_file.sh deleted file mode 100755 index 4f5b86f..0000000 --- a/ci/upload_release_file.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -set -e - -. $HOME/.cargo/env - -if [[ "$GITHUB_TOKEN" == "" ]]; then - echo "GitHub access token GITHUB_TOKEN env is not provided!" - exit 1 -fi - -RETRIES=0 -until [ $RETRIES -eq 20 ] -do - echo "Finding the GitHub release associated with '$CIRRUS_TAG' tag..." - CIRRUS_RELEASE=$(curl -sL -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases | jq -c "[ .[] | select( .tag_name | contains(\"$CIRRUS_TAG\")) ] | .[0]" | jq -r '.id') - [[ "$CIRRUS_RELEASE" != "null" ]] && break - RETRIES=$((RETRIES+1)) - sleep 30 -done - -if [[ "$CIRRUS_RELEASE" == "null" ]]; then - echo "Can not find the associated GitHub '$CIRRUS_TAG' release!" - exit 1 -fi - -echo "GitHub release '$CIRRUS_TAG' found. Preparing asset files to upload..." - -files_to_upload="static-web-server-$CIRRUS_TAG-$BINARY_ARCH-unknown-freebsd.tar.gz" - -echo "Uploading GitHub release asset '$files_to_upload'..." -name=$(basename "$files_to_upload") -url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_RELEASE/assets?name=$name" -curl -LX POST \ - --data-binary @$files_to_upload \ - --header "Authorization: token $GITHUB_TOKEN" \ - --header "Content-Type: application/octet-stream" \ - $url_to_upload - -echo -echo "GitHub release '$CIRRUS_TAG' assets uploaded successfully." diff --git a/scripts/ci/cargo.sh b/scripts/ci/cargo.sh new file mode 100755 index 0000000..23efb4e --- /dev/null +++ b/scripts/ci/cargo.sh @@ -0,0 +1,63 @@ +# Adapted from https://github.com/briansmith/ring/blob/main/mk/cargo.sh + +set -eux -o pipefail + +OS_NAME=$(uname -s | tr A-Z a-z) + +# Avoid putting the Android tools in `$PATH` because there are tools in this +# directory like `clang` that would conflict with the same-named tools that may +# be needed to compile the build script, or to compile for other targets. +if [ -n "${ANDROID_HOME-}" ]; then + # Keep the next line in sync with the corresponding line in install-build-tools.sh. + ndk_version=25.2.9519653 + ANDROID_NDK_ROOT=${ANDROID_NDK_ROOT:-${ANDROID_HOME}/ndk/$ndk_version} +fi +if [ -n "${ANDROID_NDK_ROOT-}" ]; then + android_tools=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/${OS_NAME}-x86_64/bin +fi + +for arg in $*; do + case $arg in + --target=*) + target=${arg#*=} + ;; + *) + ;; + esac +done + +case $target in + aarch64-linux-android) + export CC_aarch64_linux_android=$android_tools/aarch64-linux-android21-clang + export AR_aarch64_linux_android=$android_tools/llvm-ar + export CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER=$android_tools/aarch64-linux-android21-clang + ;; + # aarch64-unknown-linux-gnu) + # export CC_aarch64_unknown_linux_gnu=clang-$llvm_version + # export AR_aarch64_unknown_linux_gnu=llvm-ar-$llvm_version + # export CFLAGS_aarch64_unknown_linux_gnu="--sysroot=/usr/aarch64-linux-gnu" + # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc + # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="$qemu_aarch64" + # ;; + # aarch64-unknown-linux-musl) + # export CC_aarch64_unknown_linux_musl=clang-$llvm_version + # export AR_aarch64_unknown_linux_musl=llvm-ar-$llvm_version + # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="$rustflags_self_contained" + # export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER="$qemu_aarch64" + # ;; + # arm-unknown-linux-gnueabihf) + # export CC_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc + # export AR_arm_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc-ar + # export CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc + # export CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_RUNNER="$qemu_arm" + # ;; + armv7-linux-androideabi) + export CC_armv7_linux_androideabi=$android_tools/armv7a-linux-androideabi19-clang + export AR_armv7_linux_androideabi=$android_tools/llvm-ar + export CARGO_TARGET_ARMV7_LINUX_ANDROIDEABI_LINKER=$android_tools/armv7a-linux-androideabi19-clang + ;; + *) + ;; +esac + +cargo "$@" diff --git a/scripts/ci/get_latest_release.sh b/scripts/ci/get_latest_release.sh new file mode 100755 index 0000000..022605b --- /dev/null +++ b/scripts/ci/get_latest_release.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +echo "Getting latest release number..." + +curl \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/static-web-server/static-web-server/releases?per_page=5&page=1" | \ +jq -c "[ .[] | select( .tag_name | contains(\"v2.\")) ] | .[0]" | \ +jq -r ".tag_name" > /tmp/version + +version=$(cat /tmp/version) + +echo "SERVER_VERSION=${version#*v}" > /tmp/version + +echo "Version saved on '/tmp/version'" diff --git a/scripts/ci/install_tools.sh b/scripts/ci/install_tools.sh new file mode 100755 index 0000000..d1dee42 --- /dev/null +++ b/scripts/ci/install_tools.sh @@ -0,0 +1,36 @@ +# Adapted from https://github.com/briansmith/ring/blob/main/mk/install-build-tools.sh + +set -eux -o pipefail + +target=$1 + +case $target in +--target*android*) + # https://blog.rust-lang.org/2023/01/09/android-ndk-update-r25.html says + # "Going forward the Android platform will target the most recent LTS NDK, + # allowing Rust developers to access platform features sooner. These updates + # should occur yearly and will be announced in release notes." Assume that + # means that we should always prefer to be using the latest 25.x.y version of + # the NDK until the Rust project announces that we should use a higher major + # version number. + # + # TODO: This should probably be implemented as a map of Rust toolchain version + # to NDK version; e.g. our MSRV might (only) support an older NDK than the + # latest stable Rust toolchain. + # + # Keep the following line in sync with the corresponding line in cargo.sh. + ndk_version=25.2.9519653 + + mkdir -p "${ANDROID_HOME}/licenses" + android_license_file="${ANDROID_HOME}/licenses/android-sdk-license" + accept_android_license=24333f8a63b6825ea9c5514f83c2829b004d1fee + grep --quiet --no-messages "$accept_android_license" "$android_license_file" \ + || echo $accept_android_license >> "$android_license_file" + "${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" + + # XXX: Older Rust toolchain versions link with `-lgcc` instead of `-lunwind`; + # see https://github.com/rust-lang/rust/pull/85806. + find -L ${ANDROID_NDK_ROOT:-${ANDROID_HOME}/ndk/$ndk_version} -name libunwind.a \ + -execdir sh -c 'echo "INPUT(-lunwind)" > libgcc.a' \; + ;; +esac diff --git a/scripts/ci/upload_release_file.sh b/scripts/ci/upload_release_file.sh new file mode 100755 index 0000000..4f5b86f --- /dev/null +++ b/scripts/ci/upload_release_file.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -e + +. $HOME/.cargo/env + +if [[ "$GITHUB_TOKEN" == "" ]]; then + echo "GitHub access token GITHUB_TOKEN env is not provided!" + exit 1 +fi + +RETRIES=0 +until [ $RETRIES -eq 20 ] +do + echo "Finding the GitHub release associated with '$CIRRUS_TAG' tag..." + CIRRUS_RELEASE=$(curl -sL -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases | jq -c "[ .[] | select( .tag_name | contains(\"$CIRRUS_TAG\")) ] | .[0]" | jq -r '.id') + [[ "$CIRRUS_RELEASE" != "null" ]] && break + RETRIES=$((RETRIES+1)) + sleep 30 +done + +if [[ "$CIRRUS_RELEASE" == "null" ]]; then + echo "Can not find the associated GitHub '$CIRRUS_TAG' release!" + exit 1 +fi + +echo "GitHub release '$CIRRUS_TAG' found. Preparing asset files to upload..." + +files_to_upload="static-web-server-$CIRRUS_TAG-$BINARY_ARCH-unknown-freebsd.tar.gz" + +echo "Uploading GitHub release asset '$files_to_upload'..." +name=$(basename "$files_to_upload") +url_to_upload="https://uploads.github.com/repos/$CIRRUS_REPO_FULL_NAME/releases/$CIRRUS_RELEASE/assets?name=$name" +curl -LX POST \ + --data-binary @$files_to_upload \ + --header "Authorization: token $GITHUB_TOKEN" \ + --header "Content-Type: application/octet-stream" \ + $url_to_upload + +echo +echo "GitHub release '$CIRRUS_TAG' assets uploaded successfully." -- libgit2 1.7.2