diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml new file mode 100644 index 000000000..bc4bd4f4a --- /dev/null +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -0,0 +1,82 @@ +name: Build Model from Upstream + +env: + BUILD_DIR: "/data/github/openpilot" + OUTPUT_DIR: ${{ github.workspace }}/output + SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache + UPSTREAM_REPO: "commaai/openpilot" + +on: + workflow_dispatch: + inputs: + upstream_branch: + description: 'Upstream branch to build from' + required: true + default: 'master' + type: string + +jobs: + build_model: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ env.UPSTREAM_REPO }} + ref: ${{ github.event.inputs.upstream_branch }} + submodules: recursive + + - run: git lfs pull + + - name: Cache SCons + uses: actions/cache@v4 + with: + path: ${{env.SCONS_CACHE_DIR}} + key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}-model-${{ github.head_ref }}-${{ github.sha }} + restore-keys: | + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}-model-${{ github.head_ref }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}-model + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }} + scons-${{ runner.os }}-${{ runner.arch }}-master-new + scons-${{ runner.os }}-${{ runner.arch }}-master + scons-${{ runner.os }}-${{ runner.arch }} + + - name: Setup build environment + run: | + mkdir -p "${BUILD_DIR}/" + sudo find $BUILD_DIR/ -mindepth 1 -delete + echo "Starting build stage..." + echo "Building from: ${{ env.UPSTREAM_REPO }} branch: ${{ github.event.inputs.upstream_branch }}" + + - name: Patch SConstruct to pass arbitrary cache + run: | + sed -i.bak 's#cache_dir =#default_cache_dir =#' ${{ github.workspace }}/SConstruct + printf '/default_cache_dir/a\\\ncache_dir = ARGUMENTS.get("cache_dir", default_cache_dir)\n' | sed -i.bak -f - ${{ github.workspace }}/SConstruct + cat ${{ github.workspace }}/SConstruct + + - name: Build Model + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + scons -j$(nproc) cache_dir=${{ env.SCONS_CACHE_DIR }} ${{ github.workspace }}/selfdrive/modeld + + - name: Prepare Output + run: | + sudo rm -rf ${OUTPUT_DIR} + mkdir -p ${OUTPUT_DIR} + rsync -avm \ + --include='*.dlc' \ + --include='*.thneed' \ + --include='*.pkl' \ + --include='*.onnx' \ + --exclude='*' \ + --delete-excluded \ + --chown=comma:comma \ + ./selfdrive/modeld/models/ ${OUTPUT_DIR}/ + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: model-${{ github.event.inputs.upstream_branch }}-${{ github.run_number }} + path: ${{ env.OUTPUT_DIR }} \ No newline at end of file diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml new file mode 100644 index 000000000..147a87d6d --- /dev/null +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -0,0 +1,251 @@ +name: sunnypilot prebuilt action + +env: + BUILD_DIR: "/data/openpilot" + OUTPUT_DIR: ${{ github.workspace }}/output + CI_DIR: ${{ github.workspace }}/release/ci + SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache + PUBLIC_REPO_URL: "https://github.com/sunnyhaibin/sunnypilot" + + # Branch configurations + MASTER_BRANCH: "master" + MASTER_NEW_BRANCH: "master-new" + DEV_C3_SOURCE_BRANCH: "master-dev-c3-new" + + # Target branch configurations + STAGING_TARGET_BRANCH: "staging-c3-new" + DEV_TARGET_BRANCH: "dev-c3-new" + RELEASE_TARGET_BRANCH: "release-c3-new" + +on: + push: + branches: [ master, master-new, master-dev-c3-new ] + tags: [ '*' ] + pull_request: + branches: [ master, master-new ] + workflow_dispatch: + inputs: + extra_version: + description: 'Extra version identifier' + required: false + default: '' + +jobs: + build: + runs-on: self-hosted + outputs: + new_branch: ${{ steps.set-env.outputs.new_branch }} + version: ${{ steps.set-env.outputs.version }} + extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }} + should_publish: ${{ steps.set-env.outputs.should_publish }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: git lfs pull + + - name: Cache SCons + uses: actions/cache@v4 + with: + path: ${{env.SCONS_CACHE_DIR}} + key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}-${{ github.head_ref }}-${{ github.sha }} + restore-keys: | + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }}-${{ github.head_ref }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }} + scons-${{ runner.os }}-${{ runner.arch }} + + - name: Configure for dev branch + if: github.ref_name == env.DEV_C3_SOURCE_BRANCH + run: | + echo "BRANCH_TYPE=dev" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV + echo "SHOULD_PUBLISH=true" >> $GITHUB_ENV + + - name: Configure for master branches + if: github.ref_name == env.MASTER_BRANCH || github.ref_name == env.MASTER_NEW_BRANCH + run: | + echo "BRANCH_TYPE=master" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV + echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')" >> $GITHUB_ENV + echo "SHOULD_PUBLISH=true" >> $GITHUB_ENV + + - name: Configure for tags + if: startsWith(github.ref, 'refs/tags/') + run: | + echo "BRANCH_TYPE=tag" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV + echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')" >> $GITHUB_ENV + echo "SHOULD_PUBLISH=true" >> $GITHUB_ENV + + - name: Configure for manual build + if: github.event_name == 'workflow_dispatch' + run: | + echo "BRANCH_TYPE=dispatch" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ github.ref_name }}-prebuilt" >> $GITHUB_ENV + echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV + echo "SHOULD_PUBLISH=true" >> $GITHUB_ENV + + - name: Set environment variables + id: set-env + run: | + # Write to GITHUB_OUTPUT from environment variables + echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT + [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT + [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "should_publish=${SHOULD_PUBLISH:-false}" >> $GITHUB_OUTPUT + + # Set up common environment + source /etc/profile; + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + printenv >> $GITHUB_ENV + if [[ "${{ runner.debug }}" == "1" ]]; then + cat $GITHUB_OUTPUT + fi + + - name: Setup build environment + run: | + mkdir -p "${BUILD_DIR}/" + sudo find $BUILD_DIR/ -mindepth 1 -delete + echo "Starting build stage..." + echo "BUILD_DIR: ${BUILD_DIR}" + echo "CI_DIR: ${CI_DIR}" + echo "VERSION: ${{ steps.set-env.outputs.version }}" + echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}" + echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" + echo "-------" + if [[ "${{ runner.debug }}" == "1" ]]; then + printenv + fi + + - name: Build Panda + run: | + scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} ${{ github.workspace }}/panda + + - name: Build Main Project + run: | + export PYTHONPATH="$BUILD_DIR" + ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/ + cd $BUILD_DIR + sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py + scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal + touch ${BUILD_DIR}/prebuilt + if [[ "${{ runner.debug }}" == "1" ]]; then + ls -la ${BUILD_DIR} + fi + + - name: Prepare Output + run: | + sudo rm -rf ${OUTPUT_DIR} + mkdir -p ${OUTPUT_DIR} + rsync -am${RUNNER_DEBUG:+v} \ + --include='**/panda/board/' \ + --include='**/panda/board/obj' \ + --include='**/panda/board/obj/panda.bin.signed' \ + --include='**/panda/board/obj/panda_h7.bin.signed' \ + --include='**/panda/board/obj/bootstub.panda.bin' \ + --include='**/panda/board/obj/bootstub.panda_h7.bin' \ + --exclude='.sconsign.dblite' \ + --exclude='*.a' \ + --exclude='*.o' \ + --exclude='*.os' \ + --exclude='*.pyc' \ + --exclude='moc_*' \ + --exclude='*.cc' \ + --exclude='Jenkinsfile' \ + --exclude='supercombo.onnx' \ + --exclude='**/panda/board/*' \ + --exclude='**/panda/board/obj/**' \ + --exclude='**/panda/certs/' \ + --exclude='**/panda/crypto/' \ + --exclude='**/release/' \ + --exclude='**/.github/' \ + --exclude='**/selfdrive/ui/replay/' \ + --exclude='**/__pycache__/' \ + --exclude='**/selfdrive/ui/*.h' \ + --exclude='**/selfdrive/ui/**/*.h' \ + --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \ + --exclude='${{env.SCONS_CACHE_DIR}}' \ + --exclude='**/.git/' \ + --exclude='**/SConstruct' \ + --exclude='**/SConscript' \ + --exclude='**/.venv/' \ + --delete-excluded \ + --chown=comma:comma \ + ${BUILD_DIR}/ ${OUTPUT_DIR}/ + + - name: 'Tar.gz files' + run: | + tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} . + ls -la prebuilt.tar.gz + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: prebuilt + path: prebuilt.tar.gz + + publish-public: + needs: build + runs-on: ubuntu-latest + if: needs.build.outputs.should_publish == 'true' + steps: + - uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish to Public Repository + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo '${{ toJSON(needs.build.outputs) }}' + ls -la ${{ env.OUTPUT_DIR }} + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "${{ needs.build.outputs.new_branch }}" \ + "${{ needs.build.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "-${{ needs.build.outputs.extra_version_identifier }}" + + notify: + needs: [ build, publish-public ] + runs-on: ubuntu-latest + if: needs.build.outputs.new_branch != '' + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl jq gettext-base + + - name: Send Discord Notification + env: + DISCORD_WEBHOOK: ${{ needs.build.outputs.new_branch == 'dev-c3' && secrets.DISCORD_NEW_BUILD_WEBHOOK_URL || secrets.DISCORD_MANUAL_BUILD_WEBHOOK_URL }} + run: | + TEMPLATE="${{ needs.build.outputs.new_branch == 'dev-c3' && 'discord_template_notify_dev_public.json' || 'discord_template_notify_dev_private.json' }}" + export EXTRA_VERSION_IDENTIFIER="${{ needs.build.outputs.extra_version_identifier }}" + export VERSION="${{ needs.build.outputs.version }}" + cat release/ci/${TEMPLATE} | envsubst > payload.json + curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK \ No newline at end of file diff --git a/SConstruct b/SConstruct index d1661e7b6..934143fa0 100644 --- a/SConstruct +++ b/SConstruct @@ -237,7 +237,8 @@ if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') # Setup cache dir -cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir) CacheDir(cache_dir) Clean(["."], cache_dir) diff --git a/release/ci/discord_template_notify_dev_private.json b/release/ci/discord_template_notify_dev_private.json new file mode 100644 index 000000000..5da1e4d40 --- /dev/null +++ b/release/ci/discord_template_notify_dev_private.json @@ -0,0 +1,13 @@ +{ + "embeds": [ + { + "title": "❗️ Action Required for `${CI_COMMIT_REF_NAME}` ❗️", + "description": "[${CI_PROJECT_NAME}](${CI_PROJECT_URL}): Pipeline [#${CI_PIPELINE_ID}](${CI_PROJECT_URL}/-/pipelines/${CI_PIPELINE_ID}) of branch [${CI_COMMIT_REF_NAME}](${CI_PROJECT_URL}/-/commits/${CI_COMMIT_REF_NAME}) by ${GITLAB_USER_NAME} (${GITLAB_USER_LOGIN}) is ready to publish manually as [${NEW_BRANCH}](${PUBLIC_REPO_URL}/tree/${NEW_BRANCH})", + "color": 16763904, + "author": { + "name": "${GITLAB_USER_LOGIN}", + "icon_url": "${AVATAR_URL}" + } + } + ] +} diff --git a/release/ci/discord_template_notify_dev_public.json b/release/ci/discord_template_notify_dev_public.json new file mode 100644 index 000000000..c933ea10b --- /dev/null +++ b/release/ci/discord_template_notify_dev_public.json @@ -0,0 +1,9 @@ +{ + "embeds": [ + { + "title": "🎉 sunnypilot `${NEW_BRANCH}` New Update 🎉", + "description": "[sunnypilot](${PUBLIC_REPO_URL}): Build #${EXTRA_VERSION_IDENTIFIER} of branch [${NEW_BRANCH}](${PUBLIC_REPO_URL}/tree/${NEW_BRANCH}) has been published.\n\nDrive safe! 🚗💨", + "color": 4321431 + } + ] +} diff --git a/release/ci/install_github_runner.sh b/release/ci/install_github_runner.sh new file mode 100755 index 000000000..3aa6b79dd --- /dev/null +++ b/release/ci/install_github_runner.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +set -e + +# Default values +DEFAULT_REPO_URL="https://github.com/sunnypilot" +START_AT_BOOT=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --start-at-boot) + START_AT_BOOT=true + shift + ;; + --token) + GITHUB_TOKEN="$2" + shift 2 + ;; + --repo) + REPO_URL="$2" + shift 2 + ;; + *) + if [ -z "$GITHUB_TOKEN" ]; then + GITHUB_TOKEN="$1" + elif [ -z "$REPO_URL" ]; then + REPO_URL="$1" + fi + shift + ;; + esac +done + +# Check required arguments +if [ -z "$GITHUB_TOKEN" ]; then + echo "Usage: $0 [--start-at-boot] [--token ] [--repo ]" + echo "Required argument: github_token" + echo "Optional arguments:" + echo " --start-at-boot Enable auto-start at boot (default: false)" + echo " --repo Repository URL (default: ${DEFAULT_REPO_URL})" + exit 1 +fi + +# Set repository URL if not provided +REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}" + +# Constants +RUNNER_USER="github-runner" +USER_GROUPS="comma,gpu,gpio,sudo" +BASE_DIR="/data/github" +RUNNER_DIR="${BASE_DIR}/runner" +BUILDS_DIR="${BASE_DIR}/builds" +LOGS_DIR="${BASE_DIR}/logs" +CACHE_DIR="${BASE_DIR}/cache" +OPENPILOT_DIR="${BASE_DIR}/openpilot" + +create_directories() { + sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" + mkdir -p "/data/openpilot" + sudo chown -R comma:comma "/data/openpilot" +} + +download_and_setup_runner() { + cd "$RUNNER_DIR" + curl -o actions-runner-linux-arm64-2.321.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-arm64-2.321.0.tar.gz + tar xzf ./actions-runner-linux-arm64-2.321.0.tar.gz + rm ./actions-runner-linux-arm64-2.321.0.tar.gz + chmod +x ./config.sh +} + +setup_runner_user() { + sudo useradd --comment 'GitHub Runner' --create-home --home-dir ${BASE_DIR} ${RUNNER_USER} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${RUNNER_USER} + export BASE_DIR + sudo -u ${RUNNER_USER} bash -c "truncate -s 0 '${BASE_DIR}/.bash_logout'" +} + +create_sudoers_entry() { + sudo grep -qxF "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers +} + +configure_runner() { + cd "$RUNNER_DIR" + sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended +} + +set_directory_permissions() { + sudo chown -R ${RUNNER_USER}:comma "$BASE_DIR" + sudo chmod g+rwx "$BASE_DIR" + sudo chmod g+s "$BASE_DIR" +} + +modify_service_template() { + cat < "$RUNNER_DIR/bin/actions.runner.service.template" +[Unit] +Description={{Description}} +After=network-online.target nss-lookup.target time-sync.target +Wants=network-online.target nss-lookup.target time-sync.target +StartLimitInterval=5 +StartLimitBurst=10 + +[Service] +Type=simple +User=root +ExecStart=/usr/bin/unshare -m -- /bin/bash -c 'mount --bind ${OPENPILOT_DIR} /data/openpilot && setpriv --reuid={{User}} --regid={{User}} --init-groups env HOME=${BASE_DIR} USER={{User}} LOGNAME={{User}} MAIL=/var/mail/{{User}} {{RunnerRoot}}/runsvc.sh' +WorkingDirectory={{RunnerRoot}} +KillMode=process +KillSignal=SIGTERM +TimeoutStopSec=5min +Restart=always +RestartSec=120 + +[Install] +WantedBy=multi-user.target +EOL +} + +# Make filesystem writable +sudo mount -o remount,rw / + +# Ensure filesystem is remounted as read-only on script exit +trap "sudo mount -o remount,ro /" EXIT + +# Execute installation steps +setup_runner_user +create_sudoers_entry +create_directories +download_and_setup_runner +modify_service_template +configure_runner +set_directory_permissions + +# Install and start service using built-in installer +cd "$RUNNER_DIR" +sudo ./svc.sh install $RUNNER_USER + +# Handle auto-start configuration +if [ "$START_AT_BOOT" = false ]; then + sudo systemctl disable actions.runner.sunnypilot.$(uname -n) +fi + +sudo ./svc.sh start \ No newline at end of file diff --git a/release/ci/publish.sh b/release/ci/publish.sh new file mode 100755 index 000000000..d72378293 --- /dev/null +++ b/release/ci/publish.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +# Take parameters as arguments +SOURCE_DIR=$1 +OUTPUT_DIR=$2 +DEV_BRANCH=$3 +VERSION=$4 +GIT_ORIGIN=$5 +EXTRA_VERSION_IDENTIFIER=$6 + +# Check parameters +if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then + echo "Error: No source or output directory provided." + exit 1 +fi + +if [ -z "$DEV_BRANCH" ] || [ -z "$VERSION" ]; then + echo "Error: No dev branch or version provided." + exit 1 +fi + +if [ -z "$GIT_ORIGIN" ]; then + echo "Error: No GIT_ORIGIN provided" + exit 1 +fi + +# "Tagging" +echo "#define COMMA_VERSION \"$VERSION\"" > ${OUTPUT_DIR}/common/version.h + +## set git identity +#source $DIR/identity.sh +#export GIT_SSH_COMMAND="ssh -i /data/gitkey" + +echo "[-] Setting up repo T=$SECONDS" +cd $OUTPUT_DIR +git init + +# set git username/password +#source /data/identity.sh + +git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .git doesn't exist or not allowed to delete +git remote remove origin || true # ensure cleanup +git remote add origin $GIT_ORIGIN +#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out +git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) + +echo "[-] committing version $VERSION T=$SECONDS" +git add -f . +git commit -a -m "sunnypilot v$VERSION release" +git branch --set-upstream-to=origin/$DEV_BRANCH + +# include source commit hash and build date in commit +GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) +DATETIME=$(date '+%Y-%m-%dT%H:%M:%S') +SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}') + +# Add built files to git +git add -f . +if [ "$EXTRA_VERSION_IDENTIFIER" = "-release" ] || [ "$EXTRA_VERSION_IDENTIFIER" = "-staging" ]; then + export VERSION=${VERSION%"$EXTRA_VERSION_IDENTIFIER"} + git commit --amend -m "sunnypilot v$VERSION" +else + git commit --amend -m "sunnypilot v$VERSION + version: sunnypilot v$SP_VERSION release + date: $DATETIME + master commit: $GIT_HASH + " +fi +git branch -m $DEV_BRANCH + +# Push! +echo "[-] pushing T=$SECONDS" +git push -f origin $DEV_BRANCH diff --git a/release/ci/uninstall_github_runner.sh b/release/ci/uninstall_github_runner.sh new file mode 100755 index 000000000..a7b0ca3c0 --- /dev/null +++ b/release/ci/uninstall_github_runner.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Define directories and user +GITHUB_BASE_DIR="/data/github" +BIN_DIR="$GITHUB_BASE_DIR/bin" +BUILDS_DIR="$GITHUB_BASE_DIR/builds" +OPENPILOT_DIR="$GITHUB_BASE_DIR/openpilot" +LOGS_DIR="$GITHUB_BASE_DIR/logs" +CACHE_DIR="$GITHUB_BASE_DIR/cache" +RUNNER_USERNAME="github-runner" +# Define the systemd service name +SERVICE_NAME="github-runner" +USER_GROUPS="comma,gpu,gpio,sudo" + +# Function to stop and disable the systemd service +stop_and_uninstall_service() { + cd $GITHUB_BASE_DIR/runner + sudo ./svc.sh stop + sudo ./svc.sh uninstall +} + +# Function to remove the systemd service file +remove_runner() { + cd $GITHUB_BASE_DIR/runner + sudo rm .runner + sudo su -c './config.sh remove' github-runner +} + +# Function to delete the Github Runner directories +delete_directories() { + sudo rm -rf "$BIN_DIR/github-runner" + sudo rm -rf "$GITHUB_BASE_DIR" "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" +} + +# Function to remove the Github Runner user +delete_user() { + for group in ${USER_GROUPS//,/ } + do + sudo gpasswd -d ${RUNNER_USERNAME} ${group} + done + sudo userdel -r ${RUNNER_USERNAME} +} + +# Function to remove sudoers entry +remove_sudoers_entry() { + sudo sed -i.bak "/${RUNNER_USERNAME} ALL=(ALL) NOPASSWD: ALL/d" /etc/sudoers +} + +# Make filesystem writable +sudo mount -o remount rw / + +# Ensure filesystem is remounted as read-only on script exit +trap "sudo mount -o remount ro /" EXIT + +# Call functions +stop_and_uninstall_service +remove_runner +delete_directories +delete_user +remove_sudoers_entry +# End of uninstall script diff --git a/release/release_files.py b/release/release_files.py index 0e1ed852a..702255f96 100755 --- a/release/release_files.py +++ b/release/release_files.py @@ -32,6 +32,7 @@ blacklist = [ ".git/", ".github/", + ".devcontainer/", "Darwin/", ".vscode", @@ -47,6 +48,43 @@ blacklist = [ ".gitmodules", ] +# Sunnypilot blacklist +sunnypilot_blacklist = [ + "system/loggerd/sunnylink_uploader.py", # Temporarily, until we are ready to roll it out widely + "system/manager/gitlab_runner.sh", + ".idea/", + ".run/", + ".*__pycache__/.*", + ".*\\.pyc", + "teleoprtc/*", + "third_party/snpe/x86_64/*", + "body/board/canloader.py", + "body/board/flash_base.sh", + "body/board/flash_knee.sh", + "body/board/recover.sh", + ".*/test/", + ".*/tests/", + ".*tinygrad_repo/tinygrad/renderer/", + "README.md", + ".*internal/", + "docs/.*", + ".sconsign.dblite", + "release/ci/scons_cache/", + ".gitlab-ci.yml", + ".clang-tidy", + ".dockerignore", + ".editorconfig", + ".python-version", + "SECURITY.md", + "codecov.yml", + "conftest.py", + "poetry.lock", + ".venv/", +] + +# Merge the blacklists +blacklist += sunnypilot_blacklist + # gets you through the blacklist whitelist = [ "tools/lib/", @@ -119,8 +157,45 @@ whitelist = [ "opendbc_repo/dbc/toyota_tss2_adas.dbc", "opendbc_repo/dbc/vw_golf_mk4.dbc", "opendbc_repo/dbc/vw_mqb_2010.dbc", + "opendbc_repo/dbc/tesla_can.dbc", + "opendbc_repo/dbc/tesla_radar_bosch_generated.dbc", + "opendbc_repo/dbc/tesla_radar_continental_generated.dbc", + "opendbc_repo/dbc/tesla_powertrain.dbc", ] +# Sunnypilot whitelist +sunnypilot_whitelist = [ + "^README.md", + ".*selfdrive/test/fuzzy_generation.py", + ".*selfdrive/test/helpers.py", + ".*selfdrive/test/__init__.py", + ".*selfdrive/test/setup_device_ci.sh", + ".*selfdrive/test/test_time_to_onroad.py", + ".*selfdrive/test/test_onroad.py", + ".*system/manager/test/test_manager.py", + ".*system/manager/test/__init__.py", + ".*system/qcomgpsd/tests/test_qcomgpsd.py", + ".*system/updated/casync/tests/test_casync.py", + ".*system/updated/tests/test_git.py", + ".*system/updated/tests/test_base.py", + ".*selfdrive/ui/tests/test_translations.py", + ".*selfdrive/car/tests/__init__.py", + ".*selfdrive/car/tests/test_car_interfaces.py", + ".*selfdrive/navd/tests/test_navd.py", + ".*selfdrive/navd/tests/test_map_renderer.py", + ".*selfdrive/boardd/tests/test_boardd_loopback.py", + ".*INTEGRATION.md", + ".*HOW-TOS.md", + ".*CARS.md", + ".*LIMITATIONS.md", + ".*CONTRIBUTING.md", + ".*sunnyhaibin0850_qrcode_paypal.me.png", + "opendbc/.*.dbc", +] + +# Merge the whitelists +whitelist += sunnypilot_whitelist + if __name__ == "__main__": for f in Path(ROOT).rglob("**/*"):