diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 6bb55716a2..d0e3486020 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -95,7 +95,6 @@ jobs: timeout-minutes: 30 build_mac: - if: github.repository == 'commaai/openpilot' # Blocking macos builds as well since they have a 10x miltiplier for GH action minutes, waaaay too much! name: build macOS runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml new file mode 100644 index 0000000000..9b700e186f --- /dev/null +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -0,0 +1,86 @@ +name: Build Model from Upstream + +env: + BUILD_DIR: "/data/openpilot" + OUTPUT_DIR: ${{ github.workspace }}/output + SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache + UPSTREAM_REPO: "commaai/openpilot" + +on: + workflow_dispatch: + inputs: + upstream_branch: + description: 'Upstream branch to build from' + required: true + default: 'master' + type: string + custom_name: + description: 'Custom name for the model' + required: false + type: string + +jobs: + build_model: + runs-on: self-hosted + + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ env.UPSTREAM_REPO }} + ref: ${{ github.event.inputs.upstream_branch }} + submodules: recursive + + - run: git lfs pull + + - name: Cache SCons + uses: actions/cache@v4 + with: + path: ${{env.SCONS_CACHE_DIR}} + key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }} + restore-keys: | + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }} + scons-${{ runner.os }}-${{ runner.arch }}-master-new + scons-${{ runner.os }}-${{ runner.arch }}-master + scons-${{ runner.os }}-${{ runner.arch }} + + - name: Setup build environment + run: | + mkdir -p "${BUILD_DIR}/" + sudo find $BUILD_DIR/ -mindepth 1 -delete + echo "Starting build stage..." + echo "Building from: ${{ env.UPSTREAM_REPO }} branch: ${{ github.event.inputs.upstream_branch }}" + + - name: Patch SConstruct to pass arbitrary cache + run: | + sed -i.bak 's#cache_dir =#default_cache_dir =#' ${{ github.workspace }}/SConstruct + printf '/default_cache_dir/a\\\ncache_dir = ARGUMENTS.get("cache_dir", default_cache_dir)\n' | sed -i.bak -f - ${{ github.workspace }}/SConstruct + cat ${{ github.workspace }}/SConstruct + + - name: Build Model + run: | + source /etc/profile + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + scons -j$(nproc) cache_dir=${{ env.SCONS_CACHE_DIR }} ${{ github.workspace }}/selfdrive/modeld + + - name: Prepare Output + run: | + sudo rm -rf ${OUTPUT_DIR} + mkdir -p ${OUTPUT_DIR} + rsync -avm \ + --include='*.dlc' \ + --include='*.thneed' \ + --include='*.pkl' \ + --include='*.onnx' \ + --exclude='*' \ + --delete-excluded \ + --chown=comma:comma \ + ./selfdrive/modeld/models/ ${OUTPUT_DIR}/ + + - name: Upload Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: model-${{ github.event.inputs.custom_name || github.event.inputs.upstream_branch }}-${{ github.run_number }} + path: ${{ env.OUTPUT_DIR }} diff --git a/.github/workflows/sunnypilot-build-prebuilt.yaml b/.github/workflows/sunnypilot-build-prebuilt.yaml new file mode 100644 index 0000000000..1c450425d7 --- /dev/null +++ b/.github/workflows/sunnypilot-build-prebuilt.yaml @@ -0,0 +1,268 @@ +name: sunnypilot prebuilt action + +env: + BUILD_DIR: "/data/openpilot" + OUTPUT_DIR: ${{ github.workspace }}/output + CI_DIR: ${{ github.workspace }}/release/ci + SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache + PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot" + + # Branch configurations + MASTER_BRANCH: "master" + MASTER_NEW_BRANCH: "master-new" + DEV_C3_SOURCE_BRANCH: "master-dev-c3-new" + + # Target branch configurations + STAGING_TARGET_BRANCH: "staging-c3-new" + DEV_TARGET_BRANCH: "dev-c3-new" + RELEASE_TARGET_BRANCH: "release-c3-new" + +on: + push: + branches: [ master, master-new, master-dev-c3-new ] + tags: [ '*' ] + pull_request: + branches: [ master, master-new ] + workflow_dispatch: + inputs: + extra_version: + description: 'Extra version identifier' + required: false + default: '' + +jobs: + build: + concurrency: + group: build-${{ github.head_ref || github.ref_name }} + cancel-in-progress: false + runs-on: self-hosted + outputs: + new_branch: ${{ steps.set-env.outputs.new_branch }} + version: ${{ steps.set-env.outputs.version }} + extra_version_identifier: ${{ steps.set-env.outputs.extra_version_identifier }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - run: git lfs pull + + - name: Cache SCons + uses: actions/cache@v4 + with: + path: ${{env.SCONS_CACHE_DIR}} + key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-${{ github.sha }} + restore-keys: | + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ github.ref_name }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }} + scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }} + scons-${{ runner.os }}-${{ runner.arch }} + + - name: Set Configuration + run: | + if [[ "${{ github.ref_name }}" == "${{ env.DEV_C3_SOURCE_BRANCH }}" ]]; then + # Dev configuration + echo "BRANCH_TYPE=dev" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=${{ github.run_number }}" >> $GITHUB_ENV + + elif [[ "${{ github.ref_name }}" == "${{ env.MASTER_BRANCH }}" || "${{ github.ref_name }}" == "${{ env.MASTER_NEW_BRANCH }}" ]]; then + # Master configuration + echo "BRANCH_TYPE=master" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV + echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-staging" >> $GITHUB_ENV + + elif [[ "${{ github.ref }}" == refs/tags/* ]]; then + # Tag configuration + echo "BRANCH_TYPE=tag" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV + echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV + echo "VERSION=$(cat common/version.h | grep COMMA_VERSION | sed -e 's/[^0-9|.]//g')-release" >> $GITHUB_ENV + + else + # Feature branch configuration + echo "BRANCH_TYPE=dispatch" >> $GITHUB_ENV + echo "NEW_BRANCH=${{ github.head_ref || github.ref_name }}-prebuilt" >> $GITHUB_ENV + echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV + fi + + - name: Set environment variables + id: set-env + run: | + # Write to GITHUB_OUTPUT from environment variables + echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT + [[ ! -z "$EXTRA_VERSION_IDENTIFIER" ]] && echo "extra_version_identifier=$EXTRA_VERSION_IDENTIFIER" >> $GITHUB_OUTPUT + [[ ! -z "$VERSION" ]] && echo "version=$VERSION" >> $GITHUB_OUTPUT + + # Set up common environment + source /etc/profile; + export UV_PROJECT_ENVIRONMENT=${HOME}/venv + export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + printenv >> $GITHUB_ENV + if [[ "${{ runner.debug }}" == "1" ]]; then + cat $GITHUB_OUTPUT + fi + + - name: Setup build environment + run: | + mkdir -p "${BUILD_DIR}/" + sudo find $BUILD_DIR/ -mindepth 1 -delete + echo "Starting build stage..." + echo "BUILD_DIR: ${BUILD_DIR}" + echo "CI_DIR: ${CI_DIR}" + echo "VERSION: ${{ steps.set-env.outputs.version }}" + echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}" + echo "VIRTUAL_ENV: ${VIRTUAL_ENV}" + echo "-------" + if [[ "${{ runner.debug }}" == "1" ]]; then + printenv + fi + + - name: Build Panda + run: | + scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} ${{ github.workspace }}/panda + + - name: Build Main Project + run: | + export PYTHONPATH="$BUILD_DIR" + ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/ + cd $BUILD_DIR + sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py + scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal + touch ${BUILD_DIR}/prebuilt + if [[ "${{ runner.debug }}" == "1" ]]; then + ls -la ${BUILD_DIR} + fi + + - name: Prepare Output + run: | + sudo rm -rf ${OUTPUT_DIR} + mkdir -p ${OUTPUT_DIR} + rsync -am${RUNNER_DEBUG:+v} \ + --include='**/panda/board/' \ + --include='**/panda/board/obj' \ + --include='**/panda/board/obj/panda.bin.signed' \ + --include='**/panda/board/obj/panda_h7.bin.signed' \ + --include='**/panda/board/obj/bootstub.panda.bin' \ + --include='**/panda/board/obj/bootstub.panda_h7.bin' \ + --exclude='.sconsign.dblite' \ + --exclude='*.a' \ + --exclude='*.o' \ + --exclude='*.os' \ + --exclude='*.pyc' \ + --exclude='moc_*' \ + --exclude='*.cc' \ + --exclude='Jenkinsfile' \ + --exclude='supercombo.onnx' \ + --exclude='**/panda/board/*' \ + --exclude='**/panda/board/obj/**' \ + --exclude='**/panda/certs/' \ + --exclude='**/panda/crypto/' \ + --exclude='**/release/' \ + --exclude='**/.github/' \ + --exclude='**/selfdrive/ui/replay/' \ + --exclude='**/__pycache__/' \ + --exclude='**/selfdrive/ui/*.h' \ + --exclude='**/selfdrive/ui/**/*.h' \ + --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' \ + --exclude='${{env.SCONS_CACHE_DIR}}' \ + --exclude='**/.git/' \ + --exclude='**/SConstruct' \ + --exclude='**/SConscript' \ + --exclude='**/.venv/' \ + --delete-excluded \ + --chown=comma:comma \ + ${BUILD_DIR}/ ${OUTPUT_DIR}/ + + - name: 'Tar.gz files' + run: | + tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} . + ls -la prebuilt.tar.gz + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: prebuilt + path: prebuilt.tar.gz + + publish: + concurrency: + group: publish-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + if: ${{ github.event_name != 'pull_request' || github.event_name == 'pull_request' && github.event.pull_request.draft }} + needs: build + runs-on: ubuntu-24.04 + environment: ${{ contains(fromJSON(vars.AUTO_DEPLOY_PREBUILT_BRANCHES), github.head_ref || github.ref_name) && 'auto-deploy' || 'feature-branch' }} + steps: + - uses: actions/checkout@v4 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: prebuilt + + - name: Untar prebuilt + run: | + mkdir -p ${{ env.OUTPUT_DIR }} + tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + + - name: Publish to Public Repository + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo '${{ toJSON(needs.build.outputs) }}' + ls -la ${{ env.OUTPUT_DIR }} + + ${{ env.CI_DIR }}/publish.sh \ + "${{ github.workspace }}" \ + "${{ env.OUTPUT_DIR }}" \ + "${{ needs.build.outputs.new_branch }}" \ + "${{ needs.build.outputs.version }}" \ + "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \ + "-${{ needs.build.outputs.extra_version_identifier }}" + + echo "" + echo "---- ℹ️ To update the list of branches that auto deploy prebuilts -----" + echo "" + echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES" + echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}" + echo "3. Update as needed (JSON array with no spaces)" + + notify: + needs: [ build, publish ] + runs-on: ubuntu-24.04 + if: success() + steps: + - uses: actions/checkout@v4 + - name: Setup Alpine Linux environment + uses: jirutka/setup-alpine@v1.2.0 + with: + packages: 'jq gettext curl' + + - name: Send Discord Notification + env: + DISCORD_WEBHOOK: ${{ contains(fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES), github.head_ref || github.ref_name) && secrets.DISCORD_DEV_FEEDBACK_CHANNEL_WEBHOOK || secrets.DISCORD_DEV_PRIVATE_CHANNEL_WEBHOOK }} + run: | + TEMPLATE='${{ vars.DISCORD_GENERAL_UPDATE_NOTICE }}' + export EXTRA_VERSION_IDENTIFIER="${{ needs.build.outputs.extra_version_identifier }}" + export VERSION="${{ needs.build.outputs.version }}" + export branch_name=${{ github.head_ref || github.ref_name }} + export new_branch=${{ needs.build.outputs.new_branch }} + export extra_version_identifier=${{ needs.build.outputs.extra_version_identifier || github.run_number}} + echo ${TEMPLATE} | envsubst | jq -c '.' | tee payload.json + curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK + + echo "" + echo "---- ℹ️ To update the list of branches that notify to dev-feedback -----" + echo "" + echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/DEV_FEEDBACK_NOTIFICATION_BRANCHES" + echo "2. Current value: ${{ vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES }}" + echo "3. Update as needed (JSON array with no spaces)" + shell: alpine.sh {0} \ No newline at end of file diff --git a/SConstruct b/SConstruct index d1661e7b6c..934143fa08 100644 --- a/SConstruct +++ b/SConstruct @@ -237,7 +237,8 @@ if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') # Setup cache dir -cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir) CacheDir(cache_dir) Clean(["."], cache_dir) diff --git a/common/params.cc b/common/params.cc index e13e270f28..1e845635d9 100644 --- a/common/params.cc +++ b/common/params.cc @@ -207,6 +207,7 @@ std::unordered_map keys = { {"MadsPauseLateralOnBrake", PERSISTENT}, {"MadsUnifiedEngagementMode", PERSISTENT}, {"HyundaiLongitudinalMainCruiseToggleable", PERSISTENT}, + {"EnableGithubRunner", PERSISTENT}, }; } // namespace diff --git a/release/ci/install_github_runner.sh b/release/ci/install_github_runner.sh new file mode 100755 index 0000000000..1b138996d6 --- /dev/null +++ b/release/ci/install_github_runner.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -e + +# Default values +DEFAULT_REPO_URL="https://github.com/sunnypilot" +START_AT_BOOT=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --start-at-boot) + START_AT_BOOT=true + shift + ;; + --token) + GITHUB_TOKEN="$2" + shift 2 + ;; + --repo) + REPO_URL="$2" + shift 2 + ;; + *) + if [ -z "$GITHUB_TOKEN" ]; then + GITHUB_TOKEN="$1" + elif [ -z "$REPO_URL" ]; then + REPO_URL="$1" + fi + shift + ;; + esac +done + +# Check required arguments +if [ -z "$GITHUB_TOKEN" ]; then + echo "Usage: $0 [--start-at-boot] [--token ] [--repo ]" + echo "Required argument: github_token" + echo "Optional arguments:" + echo " --start-at-boot Enable auto-start at boot (default: false)" + echo " --repo Repository URL (default: ${DEFAULT_REPO_URL})" + exit 1 +fi + +# Set repository URL if not provided +REPO_URL="${REPO_URL:-$DEFAULT_REPO_URL}" + +# Determine BASE_DIR based on mount point +if mountpoint -q /data/media; then + BASE_DIR="/data/media/0/github" +else + BASE_DIR="/data/github" +fi + +# Constants +RUNNER_USER="github-runner" +USER_GROUPS="comma,gpu,gpio,sudo" +RUNNER_DIR="${BASE_DIR}/runner" +BUILDS_DIR="${BASE_DIR}/builds" +LOGS_DIR="${BASE_DIR}/logs" +CACHE_DIR="${BASE_DIR}/cache" +OPENPILOT_DIR="${BASE_DIR}/openpilot" + +create_directories() { + sudo mkdir -p "$RUNNER_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" + mkdir -p "/data/openpilot" + sudo chown -R comma:comma "/data/openpilot" +} + +download_and_setup_runner() { + cd "$RUNNER_DIR" + curl -o actions-runner-linux-arm64-2.321.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.321.0/actions-runner-linux-arm64-2.321.0.tar.gz + tar xzf ./actions-runner-linux-arm64-2.321.0.tar.gz + rm ./actions-runner-linux-arm64-2.321.0.tar.gz + chmod +x ./config.sh +} + +setup_runner_user() { + sudo useradd --comment 'GitHub Runner' --create-home --home-dir ${BASE_DIR} ${RUNNER_USER} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${RUNNER_USER} + export BASE_DIR + sudo -u ${RUNNER_USER} bash -c "truncate -s 0 '${BASE_DIR}/.bash_logout'" +} + +create_sudoers_entry() { + sudo grep -qxF "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${RUNNER_USER} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers +} + +configure_runner() { + cd "$RUNNER_DIR" + sudo -u ${RUNNER_USER} ./config.sh --url "$REPO_URL" --token "$GITHUB_TOKEN" --name $(hostname) --runnergroup "tici-tizi" --labels "tici" --work "$BUILDS_DIR" --unattended +} + +set_directory_permissions() { + sudo chown -R ${RUNNER_USER}:comma "$BASE_DIR" + sudo chmod g+rwx "$BASE_DIR" + sudo chmod g+s "$BASE_DIR" +} + +modify_service_template() { + cat < "$RUNNER_DIR/bin/actions.runner.service.template" +[Unit] +Description={{Description}} +After=network-online.target nss-lookup.target time-sync.target +Wants=network-online.target nss-lookup.target time-sync.target +StartLimitInterval=5 +StartLimitBurst=10 + +[Service] +Type=simple +User=root +ExecStart=/usr/bin/unshare -m -- /bin/bash -c 'mount --bind ${OPENPILOT_DIR} /data/openpilot && setpriv --reuid={{User}} --regid={{User}} --init-groups env HOME=${BASE_DIR} USER={{User}} LOGNAME={{User}} MAIL=/var/mail/{{User}} {{RunnerRoot}}/runsvc.sh' +WorkingDirectory={{RunnerRoot}} +KillMode=process +KillSignal=SIGTERM +TimeoutStopSec=5min +Restart=always +RestartSec=120 + +[Install] +WantedBy=multi-user.target +EOL +} + +# Make filesystem writable +sudo mount -o remount,rw / + +# Ensure filesystem is remounted as read-only on script exit +trap "sudo mount -o remount,ro /" EXIT + +# Execute installation steps +setup_runner_user +create_sudoers_entry +create_directories +download_and_setup_runner +modify_service_template +configure_runner +set_directory_permissions + +# Install and start service using built-in installer +cd "$RUNNER_DIR" +sudo ./svc.sh install $RUNNER_USER + +# Handle auto-start configuration +if [ "$START_AT_BOOT" = false ]; then + sudo systemctl disable actions.runner.sunnypilot.$(uname -n) +fi + +sudo ./svc.sh start \ No newline at end of file diff --git a/release/ci/publish.sh b/release/ci/publish.sh new file mode 100755 index 0000000000..d723782934 --- /dev/null +++ b/release/ci/publish.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +# Take parameters as arguments +SOURCE_DIR=$1 +OUTPUT_DIR=$2 +DEV_BRANCH=$3 +VERSION=$4 +GIT_ORIGIN=$5 +EXTRA_VERSION_IDENTIFIER=$6 + +# Check parameters +if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then + echo "Error: No source or output directory provided." + exit 1 +fi + +if [ -z "$DEV_BRANCH" ] || [ -z "$VERSION" ]; then + echo "Error: No dev branch or version provided." + exit 1 +fi + +if [ -z "$GIT_ORIGIN" ]; then + echo "Error: No GIT_ORIGIN provided" + exit 1 +fi + +# "Tagging" +echo "#define COMMA_VERSION \"$VERSION\"" > ${OUTPUT_DIR}/common/version.h + +## set git identity +#source $DIR/identity.sh +#export GIT_SSH_COMMAND="ssh -i /data/gitkey" + +echo "[-] Setting up repo T=$SECONDS" +cd $OUTPUT_DIR +git init + +# set git username/password +#source /data/identity.sh + +git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .git doesn't exist or not allowed to delete +git remote remove origin || true # ensure cleanup +git remote add origin $GIT_ORIGIN +#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out +git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) + +echo "[-] committing version $VERSION T=$SECONDS" +git add -f . +git commit -a -m "sunnypilot v$VERSION release" +git branch --set-upstream-to=origin/$DEV_BRANCH + +# include source commit hash and build date in commit +GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) +DATETIME=$(date '+%Y-%m-%dT%H:%M:%S') +SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}') + +# Add built files to git +git add -f . +if [ "$EXTRA_VERSION_IDENTIFIER" = "-release" ] || [ "$EXTRA_VERSION_IDENTIFIER" = "-staging" ]; then + export VERSION=${VERSION%"$EXTRA_VERSION_IDENTIFIER"} + git commit --amend -m "sunnypilot v$VERSION" +else + git commit --amend -m "sunnypilot v$VERSION + version: sunnypilot v$SP_VERSION release + date: $DATETIME + master commit: $GIT_HASH + " +fi +git branch -m $DEV_BRANCH + +# Push! +echo "[-] pushing T=$SECONDS" +git push -f origin $DEV_BRANCH diff --git a/release/ci/uninstall_github_runner.sh b/release/ci/uninstall_github_runner.sh new file mode 100755 index 0000000000..5f3acfbafd --- /dev/null +++ b/release/ci/uninstall_github_runner.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Determine BASE_DIR based on mount point +if mountpoint -q /data/media; then + GITHUB_BASE_DIR="/data/media/0/github" +else + GITHUB_BASE_DIR="/data/github" +fi + +# Define directories and user +BIN_DIR="$GITHUB_BASE_DIR/bin" +BUILDS_DIR="$GITHUB_BASE_DIR/builds" +OPENPILOT_DIR="$GITHUB_BASE_DIR/openpilot" +LOGS_DIR="$GITHUB_BASE_DIR/logs" +CACHE_DIR="$GITHUB_BASE_DIR/cache" +RUNNER_USERNAME="github-runner" +# Define the systemd service name +SERVICE_NAME="github-runner" +USER_GROUPS="comma,gpu,gpio,sudo" + +# Function to stop and disable the systemd service +stop_and_uninstall_service() { + cd $GITHUB_BASE_DIR/runner + sudo ./svc.sh stop + sudo ./svc.sh uninstall +} + +# Function to remove the systemd service file +remove_runner() { + cd $GITHUB_BASE_DIR/runner + sudo rm .runner + sudo su -c './config.sh remove' github-runner +} + +# Function to delete the Github Runner directories +delete_directories() { + sudo rm -rf "$BIN_DIR/github-runner" + sudo rm -rf "$GITHUB_BASE_DIR" "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" +} + +# Function to remove the Github Runner user +delete_user() { + for group in ${USER_GROUPS//,/ } + do + sudo gpasswd -d ${RUNNER_USERNAME} ${group} + done + sudo userdel -r ${RUNNER_USERNAME} +} + +# Function to remove sudoers entry +remove_sudoers_entry() { + sudo sed -i.bak "/${RUNNER_USERNAME} ALL=(ALL) NOPASSWD: ALL/d" /etc/sudoers +} + +# Make filesystem writable +sudo mount -o remount rw / + +# Ensure filesystem is remounted as read-only on script exit +trap "sudo mount -o remount ro /" EXIT + +# Call functions +stop_and_uninstall_service +remove_runner +delete_directories +delete_user +remove_sudoers_entry +# End of uninstall script diff --git a/release/release_files.py b/release/release_files.py index 0e1ed852a2..dcebb19725 100755 --- a/release/release_files.py +++ b/release/release_files.py @@ -32,6 +32,7 @@ blacklist = [ ".git/", ".github/", + ".devcontainer/", "Darwin/", ".vscode", @@ -47,6 +48,42 @@ blacklist = [ ".gitmodules", ] +# Sunnypilot blacklist +sunnypilot_blacklist = [ + "system/loggerd/sunnylink_uploader.py", # Temporarily, until we are ready to roll it out widely + ".idea/", + ".run/", + ".*__pycache__/.*", + ".*\\.pyc", + "teleoprtc/*", + "third_party/snpe/x86_64/*", + "body/board/canloader.py", + "body/board/flash_base.sh", + "body/board/flash_knee.sh", + "body/board/recover.sh", + ".*/test/", + ".*/tests/", + ".*tinygrad_repo/tinygrad/renderer/", + "README.md", + ".*internal/", + "docs/.*", + ".sconsign.dblite", + "release/ci/scons_cache/", + ".gitlab-ci.yml", + ".clang-tidy", + ".dockerignore", + ".editorconfig", + ".python-version", + "SECURITY.md", + "codecov.yml", + "conftest.py", + "poetry.lock", + ".venv/", +] + +# Merge the blacklists +blacklist += sunnypilot_blacklist + # gets you through the blacklist whitelist = [ "tools/lib/", @@ -119,8 +156,45 @@ whitelist = [ "opendbc_repo/dbc/toyota_tss2_adas.dbc", "opendbc_repo/dbc/vw_golf_mk4.dbc", "opendbc_repo/dbc/vw_mqb_2010.dbc", + "opendbc_repo/dbc/tesla_can.dbc", + "opendbc_repo/dbc/tesla_radar_bosch_generated.dbc", + "opendbc_repo/dbc/tesla_radar_continental_generated.dbc", + "opendbc_repo/dbc/tesla_powertrain.dbc", ] +# Sunnypilot whitelist +sunnypilot_whitelist = [ + "^README.md", + ".*selfdrive/test/fuzzy_generation.py", + ".*selfdrive/test/helpers.py", + ".*selfdrive/test/__init__.py", + ".*selfdrive/test/setup_device_ci.sh", + ".*selfdrive/test/test_time_to_onroad.py", + ".*selfdrive/test/test_onroad.py", + ".*system/manager/test/test_manager.py", + ".*system/manager/test/__init__.py", + ".*system/qcomgpsd/tests/test_qcomgpsd.py", + ".*system/updated/casync/tests/test_casync.py", + ".*system/updated/tests/test_git.py", + ".*system/updated/tests/test_base.py", + ".*selfdrive/ui/tests/test_translations.py", + ".*selfdrive/car/tests/__init__.py", + ".*selfdrive/car/tests/test_car_interfaces.py", + ".*selfdrive/navd/tests/test_navd.py", + ".*selfdrive/navd/tests/test_map_renderer.py", + ".*selfdrive/boardd/tests/test_boardd_loopback.py", + ".*INTEGRATION.md", + ".*HOW-TOS.md", + ".*CARS.md", + ".*LIMITATIONS.md", + ".*CONTRIBUTING.md", + ".*sunnyhaibin0850_qrcode_paypal.me.png", + "opendbc/.*.dbc", +] + +# Merge the whitelists +whitelist += sunnypilot_whitelist + if __name__ == "__main__": for f in Path(ROOT).rglob("**/*"): diff --git a/selfdrive/ui/qt/offroad/developer_panel.cc b/selfdrive/ui/qt/offroad/developer_panel.cc index 5e51a38e80..710ce75d1b 100644 --- a/selfdrive/ui/qt/offroad/developer_panel.cc +++ b/selfdrive/ui/qt/offroad/developer_panel.cc @@ -24,6 +24,9 @@ DeveloperPanel::DeveloperPanel(SettingsWindow *parent) : ListWidget(parent) { }); addItem(longManeuverToggle); + auto enableGithubRunner = new ParamControl("EnableGithubRunner", tr("Enable GitHub runner service"), tr("Enables or disables the github runner service."), ""); + addItem(enableGithubRunner); + // FIXME-SP: Move to sunnypilot panels before merging auto madsToggle = new ParamControl("Mads", tr("Modular Assistive Driving System (MADS)"), "", ""); addItem(madsToggle); diff --git a/selfdrive/ui/translations/main_ar.ts b/selfdrive/ui/translations/main_ar.ts index 7797880a05..cea12a1048 100644 --- a/selfdrive/ui/translations/main_ar.ts +++ b/selfdrive/ui/translations/main_ar.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode وضع المناورة الطولية + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_de.ts b/selfdrive/ui/translations/main_de.ts index a3edba9f72..ac8b46712e 100644 --- a/selfdrive/ui/translations/main_de.ts +++ b/selfdrive/ui/translations/main_de.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_es.ts b/selfdrive/ui/translations/main_es.ts index 2afafa0113..94f7b8efaf 100644 --- a/selfdrive/ui/translations/main_es.ts +++ b/selfdrive/ui/translations/main_es.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode Modo de maniobra longitudinal + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_fr.ts b/selfdrive/ui/translations/main_fr.ts index 54b84d855d..33c7e4b918 100644 --- a/selfdrive/ui/translations/main_fr.ts +++ b/selfdrive/ui/translations/main_fr.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_ja.ts b/selfdrive/ui/translations/main_ja.ts index 22c25b9c65..51121d3174 100644 --- a/selfdrive/ui/translations/main_ja.ts +++ b/selfdrive/ui/translations/main_ja.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_ko.ts b/selfdrive/ui/translations/main_ko.ts index fa29e553c8..7854330fd2 100644 --- a/selfdrive/ui/translations/main_ko.ts +++ b/selfdrive/ui/translations/main_ko.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode 롱컨 기동 모드 + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_pt-BR.ts b/selfdrive/ui/translations/main_pt-BR.ts index c882c89e15..b1af49bf63 100644 --- a/selfdrive/ui/translations/main_pt-BR.ts +++ b/selfdrive/ui/translations/main_pt-BR.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode Modo Longitudinal Maneuver + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_th.ts b/selfdrive/ui/translations/main_th.ts index cce8a18328..c99a13567e 100644 --- a/selfdrive/ui/translations/main_th.ts +++ b/selfdrive/ui/translations/main_th.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_tr.ts b/selfdrive/ui/translations/main_tr.ts index bc44498b75..65ca1bf6f7 100644 --- a/selfdrive/ui/translations/main_tr.ts +++ b/selfdrive/ui/translations/main_tr.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_zh-CHS.ts b/selfdrive/ui/translations/main_zh-CHS.ts index d49e83efb1..be1e34ac7a 100644 --- a/selfdrive/ui/translations/main_zh-CHS.ts +++ b/selfdrive/ui/translations/main_zh-CHS.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/selfdrive/ui/translations/main_zh-CHT.ts b/selfdrive/ui/translations/main_zh-CHT.ts index 64a7a545d1..b8e69f904e 100644 --- a/selfdrive/ui/translations/main_zh-CHT.ts +++ b/selfdrive/ui/translations/main_zh-CHT.ts @@ -123,6 +123,14 @@ Longitudinal Maneuver Mode + + Enable GitHub runner service + + + + Enables or disables the github runner service. + + Modular Assistive Driving System (MADS) diff --git a/system/manager/github_runner.sh b/system/manager/github_runner.sh new file mode 100755 index 0000000000..f2170cfc70 --- /dev/null +++ b/system/manager/github_runner.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Define the service name +SERVICE_NAME="actions.runner.sunnypilot.$(uname -n)" + +# Function to control the service +control_service() { + local action=$1 # Store the function argument in a local variable + sudo systemctl $action ${SERVICE_NAME} +} + +service_exists_and_is_loaded() { + sudo systemctl status ${SERVICE_NAME} &>/dev/null + if [[ $? -ne 4 ]]; then + return 0 # Service is known to systemd (i.e., loaded) + else + return 1 # Service is unknown to systemd (i.e., not loaded) + fi +} + +# Check for required argument +if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then + echo "Usage: $0 {start|stop}" + exit 1 +fi + +# Store the script argument in a descriptive variable +ACTION=$1 + +# Trap EXIT signal (Ctrl+C) and stop the service +trap 'control_service stop ; exit' SIGINT SIGKILL EXIT + +# Enter the main loop +while true; do + # Check if the service is actually present on the system + if service_exists_and_is_loaded; then + control_service $ACTION # Call the function with the specified action + fi + sleep 1 # Pause before the next iteration +done \ No newline at end of file diff --git a/system/manager/process_config.py b/system/manager/process_config.py index a25be615f4..09b897363c 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -54,6 +54,9 @@ def only_onroad(started: bool, params: Params, CP: car.CarParams) -> bool: def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool: return not started +def use_github_runner(started, params, CP: car.CarParams) -> bool: + return not PC and params.get_bool("EnableGithubRunner") and not params.get_bool("NetworkMetered") + def or_(*fns): return lambda *args: operator.or_(*(fn(*args) for fn in fns)) @@ -111,4 +114,7 @@ procs = [ PythonProcess("joystick", "tools.joystick.joystick_control", and_(joystick, iscar)), ] +if os.path.exists("./github_runner.sh"): + procs += [NativeProcess("github_runner_start", "system/manager", ["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)] + managed_processes = {p.name: p for p in procs}