mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-09-03 06:33:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Reapply "Merge remote-tracking branch 'origin/master' into SunnyPilot"

Browse Source 
This reverts commit 42b6ecb535.
This commit is contained in:
firestar5683
2025-08-11 18:44:44 -05:00
parent f56b049010
commit 076fbbe102
318 changed files with 11962 additions and 8595 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/auto_pr_review.yaml
+7 -7
View File
@@ -1,7 +1,7 @@
name: "PR review"
on:
pull_request_target:
types: [opened, reopened, synchronize, edited]
types: [ opened, reopened, synchronize, edited ]
jobs:
labeler:
@@ -29,17 +29,17 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
target: /^(?!master-new$).*/
target: /^(?!master$).*/
exclude: /sunnypilot:.*/
change-to: ${{ github.base_ref }}
already-exists-action: close_this
already-exists-comment: "Your PR should be made against the `master-new` branch"
already-exists-comment: "Your PR should be made against the `master` branch"
update-pr-labels:
name: Update fork's PR Labels
runs-on: ubuntu-latest
if: (github.event.pull_request.head.repo.fork && (contains(github.event_name, 'pull_request') && github.event.action == 'synchronize'))
env:
env:
PR_LABEL: 'dev-c3'
TRUST_FORK_PR_LABEL: 'trust-fork-pr'
steps:
@@ -64,7 +64,7 @@ jobs:
core.setOutput('has-dev-c3', hasDevC3Label ? 'true' : 'false');
core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
- name: Remove trust-fork-pr label if present
if: steps.check-labels.outputs.has-dev-c3 == 'true' && steps.check-labels.outputs.has-trust == 'true'
uses: actions/github-script@v7
@@ -72,14 +72,14 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const prNumber = context.payload.pull_request.number;
await github.rest.issues.removeLabel({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
name: process.env.TRUST_FORK_PR_LABEL
});
console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
// Add a comment to the PR
.github/workflows/badges.yaml
+2 -2
View File
@@ -5,8 +5,8 @@ on:
workflow_dispatch:
env:
BASE_IMAGE: openpilot-base
DOCKER_REGISTRY: ghcr.io/commaai
BASE_IMAGE: sunnypilot-base
DOCKER_REGISTRY: ghcr.io/sunnypilot
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
jobs:
.github/workflows/build-all-tinygrad-models.yaml
+218 -141
View File
@@ -1,21 +1,40 @@
name: Build All Tinygrad Models and Push to GitLab
name: Build and push all tinygrad models
on:
workflow_dispatch:
inputs:
branch:
description: 'Branch to run workflow from'
required: false
default: 'master-new'
set_min_version:
description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
required: true
type: string
jobs:
setup:
runs-on: ubuntu-latest
outputs:
json_version: ${{ steps.get-json.outputs.json_version }}
recompiled_dir: ${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
json_file: ${{ steps.get-json.outputs.json_file }}
model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
tinygrad_ref: ${{ steps.get-tinygrad-ref.outputs.tinygrad_ref }}
steps:
- name: Checkout docs repo
- name: Checkout sunnypilot repo
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot
path: sunnypilot
submodules: recursive
- name: Get tinygrad_repo ref
id: get-tinygrad-ref
run: |
cd sunnypilot
export PYTHONPATH=$(pwd)
ref=$(python3 sunnypilot/models/tinygrad_ref.py)
echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
echo "tinygrad_ref is $ref"
- name: Checkout docs repo (sunnypilot-docs, gh-pages)
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot-docs
@@ -23,7 +42,7 @@ jobs:
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
- name: Get next JSON version to use
- name: Get next JSON version to use (from GitHub docs repo)
id: get-json
run: |
cd docs/docs
@@ -31,19 +50,131 @@ jobs:
next=$((latest+1))
json_file="driving_models_v${next}.json"
cp "driving_models_v${latest}.json" "$json_file"
echo "json_file=$json_file" >> $GITHUB_OUTPUT
echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
- name: Upload context for next jobs
uses: actions/upload-artifact@v4
- name: Extract tinygrad models
id: set-matrix
working-directory: docs/docs
run: |
jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
- name: Set up SSH
uses: webfactory/ssh-agent@v0.9.0
with:
name: context
path: docs
ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
- run: |
mkdir -p ~/.ssh
ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
build-all:
- name: Clone GitLab docs repo and create new recompiled dir
id: create-recompiled-dir
env:
GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
run: |
git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
cd gitlab_docs
git checkout main
git sparse-checkout set --no-cone models/
cd models
latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
if [[ -z "$latest_dir" ]]; then
next_dir=1
else
next_dir=$((latest_dir+1))
fi
recompiled_dir="${next_dir}"
mkdir -p "recompiled${recompiled_dir}"
touch "recompiled${recompiled_dir}/.gitkeep"
cd ../..
echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
- name: Push empty recompiled dir to GitLab
run: |
cd gitlab_docs
git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
git config --global user.name "GitHub Action"
git config --global user.email "action@github.com"
git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
git push origin main
- name: Push new JSON to GitHub docs repo
run: |
cd docs
git pull origin gh-pages
git add docs/"$(basename ${{ steps.get-json.outputs.json_file }})"
git config --global user.name "GitHub Action"
git config --global user.email "action@github.com"
git commit -m "Add new ${{ steps.get-json.outputs.json_file }} for build-all" || echo "No changes to commit"
git push origin gh-pages
get_and_build:
needs: [setup]
strategy:
matrix:
model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
fail-fast: false
uses: ./.github/workflows/build-single-tinygrad-model.yaml
with:
upstream_branch: ${{ matrix.model.ref }}
custom_name: ${{ matrix.model.display_name }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
secrets: inherit
retry_failed_models:
needs: [setup, get_and_build]
runs-on: ubuntu-latest
needs: setup
if: ${{ needs.setup.result != 'failure' && (failure() && !cancelled()) }}
outputs:
retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
steps:
- uses: actions/download-artifact@v4
with:
pattern: model-*
path: output
- id: set-retry-matrix
run: |
echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
built=(); while IFS= read -r line; do built+=("$line"); done < <(
ls output | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
)
jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
echo "retry_matrix=$(cat retry_matrix.json)" >> $GITHUB_OUTPUT
retry_get_and_build:
needs: [setup, get_and_build, retry_failed_models]
if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
strategy:
matrix:
model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
fail-fast: false
uses: ./.github/workflows/build-single-tinygrad-model.yaml
with:
upstream_branch: ${{ matrix.model.ref }}
custom_name: ${{ matrix.model.display_name }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
artifact_suffix: -retry
secrets: inherit
publish_models:
name: Publish models sequentially
needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
runs-on: ubuntu-latest
strategy:
max-parallel: 1
matrix:
model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
env:
JSON_FILE: docs/docs/${{ needs.setup.outputs.json_file }}
RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
JSON_FILE: ${{ needs.setup.outputs.json_file }}
ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
steps:
- name: Set up SSH
uses: webfactory/ssh-agent@v0.9.0
@@ -62,138 +193,73 @@ jobs:
echo "Cloning GitLab"
git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
cd gitlab_docs
echo "checkout models/${RECOMPILED_DIR}"
git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
git checkout main
cd ..
- name: Set next recompiled dir
id: set-recompiled
run: |
cd gitlab_docs/models
latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
if [[ -z "$latest_dir" ]]; then
next_dir=1
else
next_dir=$((latest_dir+1))
fi
recompiled_dir="recompiled${next_dir}"
mkdir -p "$recompiled_dir"
echo "RECOMPILED_DIR=$recompiled_dir" >> $GITHUB_ENV
- name: Download context
uses: actions/download-artifact@v4
- name: Checkout docs repo
uses: actions/checkout@v4
with:
name: context
path: .
repository: sunnypilot/sunnypilot-docs
ref: gh-pages
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
- name: Install dependencies
- name: Validate recompiled dir and JSON version
run: |
sudo apt-get update
sudo apt-get install -y jq gh
- name: Build all tinygrad models
id: trigger-builds
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e
> triggered_run_ids.txt
BRANCH="${{ github.event.inputs.branch }}"
jq -c '.bundles[] | select(.runner=="tinygrad")' "$JSON_FILE" | while read -r bundle; do
ref=$(echo "$bundle" | jq -r '.ref')
display_name=$(echo "$bundle" | jq -r '.display_name' | sed 's/ ([^)]*)//g')
is_20hz=$(echo "$bundle" | jq -r '.is_20hz')
echo "Triggering build for: $display_name ($ref) [20Hz: $is_20hz]"
START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
gh workflow run sunnypilot-build-model.yaml \
--repo sunnypilot/sunnypilot \
--ref "$BRANCH" \
-f upstream_branch="$ref" \
-f custom_name="$display_name" \
-f is_20hz="$is_20hz"
for i in {1..24}; do
RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="$BRANCH" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
if [ -n "$RUN_ID" ]; then
break
fi
sleep 5
done
if [ -z "$RUN_ID" ]; then
echo "ould not find the triggered workflow run for $display_name ($ref)"
exit 1
fi
echo "$RUN_ID" >> triggered_run_ids.txt
done
- name: Wait for all model builds to finish
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -e
SUCCESS_RUNS=()
FAILED_RUNS=()
declare -A RUN_ID_TO_NAME
while read -r RUN_ID; do
ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name' || echo "unknown")
RUN_ID_TO_NAME["$RUN_ID"]="$ARTIFACT_NAME"
done < triggered_run_ids.txt
while read -r RUN_ID; do
echo "Watching run ID: $RUN_ID"
gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
ARTIFACT_NAME="${RUN_ID_TO_NAME[$RUN_ID]}"
echo "Run $RUN_ID ($ARTIFACT_NAME) concluded with: $CONCLUSION"
if [[ "$CONCLUSION" == "success" ]]; then
SUCCESS_RUNS+=("$RUN_ID")
else
FAILED_RUNS+=("$RUN_ID")
fi
done < triggered_run_ids.txt
if [[ ${#SUCCESS_RUNS[@]} -eq 0 ]]; then
echo "All model builds failed. Aborting."
if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
exit 1
fi
if [ ! -f "$JSON_FILE" ]; then
echo "JSON file $JSON_FILE does not exist!"
exit 1
fi
if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
echo "WARNING: The following model builds failed:"
for RUN_ID in "${FAILED_RUNS[@]}"; do
echo "- $RUN_ID (${RUN_ID_TO_NAME[$RUN_ID]})"
done
echo "You may want to rerun these models manually."
fi
- name: Download artifact name file
uses: actions/download-artifact@v4
with:
name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
path: artifact_name
- name: Download and extract all model artifacts
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Read artifact name
id: read-artifact-name
run: |
ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
SUCCESS_RUNS=()
while read -r RUN_ID; do
CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
if [[ "$CONCLUSION" == "success" ]]; then
SUCCESS_RUNS+=("$RUN_ID")
fi
done < triggered_run_ids.txt
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
for RUN_ID in "${SUCCESS_RUNS[@]}"; do
ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
if [ -n "$ZIP_PATH" ]; then
echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
rm -f "$ZIP_PATH"
echo "Unzipped and removed $ZIP_PATH"
- name: Download model artifact
uses: actions/download-artifact@v4
with:
name: ${{ steps.read-artifact-name.outputs.artifact_name }}
path: output
- name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
run: |
find output -type f -name '*.onnx' -delete
find output -type f -name 'big_*.pkl' -delete
find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
- name: Copy model artifacts to gitlab
env:
ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
run: |
ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
mkdir -p "$ARTIFACT_DIR"
for path in output/*; do
if [ "$(basename "$path")" = "artifact_name.txt" ]; then
continue
fi
name="$(basename "$path")"
if [ -d "$path" ]; then
mkdir -p "$ARTIFACT_DIR/$name"
cp -r "$path"/* "$ARTIFACT_DIR/$name/"
echo "Copied dir $name -> $ARTIFACT_DIR/$name"
else
echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME (This is NOT an error)."
cp "$path" "$ARTIFACT_DIR/"
echo "Copied file $name -> $ARTIFACT_DIR/"
fi
echo "Done processing $ARTIFACT_NAME"
done
- name: Push recompiled dir to GitLab
@@ -202,25 +268,36 @@ jobs:
run: |
cd gitlab_docs
git checkout main
mkdir -p models/"$(basename $RECOMPILED_DIR)"
git add models/"$(basename $RECOMPILED_DIR)"
git pull origin main
for d in models/"$RECOMPILED_DIR"/*/; do
git sparse-checkout add "$d"
done
git add models/"$RECOMPILED_DIR"
git config --global user.name "GitHub Action"
git config --global user.email "action@github.com"
git commit -m "Add $(basename $RECOMPILED_DIR) from build-all-tinygrad-models"
git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
git push origin main
- run: |
cd docs
git pull origin gh-pages
- name: Run json_parser.py to update JSON
- name: update json
run: |
python3 docs/json_parser.py \
ARGS=""
[ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
ARGS="$ARGS --sort-by-date"
ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
eval python3 docs/json_parser.py \
--json-path "$JSON_FILE" \
--recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR"
--recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
$ARGS
- name: Push updated JSON to GitHub docs repo
- name: Push updated json to GitHub
run: |
cd docs
git config --global user.name "GitHub Action"
git config --global user.email "action@github.com"
git checkout gh-pages
git add docs/"$(basename $JSON_FILE)"
git commit -m "Update $(basename $JSON_FILE) after recompiling models" || echo "No changes to commit"
git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
git push origin gh-pages
.github/workflows/build-single-tinygrad-model.yaml
+107 -77
View File
@@ -1,13 +1,8 @@
name: Build Single Tinygrad Model and Push
on:
workflow_dispatch:
workflow_call:
inputs:
build_model_ref:
description: 'Branch to use for build-model workflow'
required: false
default: 'master-new'
type: string
upstream_branch:
description: 'Upstream commit to build from'
required: true
@@ -19,17 +14,47 @@ on:
recompiled_dir:
description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
required: true
type: number
type: string
json_version:
description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
required: true
type: number
type: string
artifact_suffix:
description: 'Suffix for artifact name'
required: false
type: string
default: ''
bypass_push:
description: 'Bypass pushing to GitLab for build-all'
required: false
default: true
type: boolean
workflow_dispatch:
inputs:
upstream_branch:
description: 'Upstream commit to build from'
required: true
type: string
custom_name:
description: 'Custom name for the model (no date, only name)'
required: false
type: string
recompiled_dir:
description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
required: true
type: string
json_version:
description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
required: true
type: string
model_folder:
description: 'Model folder'
required: true
type: choice
default: 'None'
options:
- None
- Simple Plan Models
- Space Lab Models
- TR Models
- DTR Models
- Custom Merge Models
@@ -42,18 +67,32 @@ on:
generation:
description: 'Model generation'
required: false
type: number
type: string
version:
description: 'Minimum selector version'
required: false
type: number
type: string
env:
RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
jobs:
build-single:
build_model:
uses: ./.github/workflows/sunnypilot-build-model.yaml
with:
upstream_branch: ${{ inputs.upstream_branch }}
custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
is_20hz: true
artifact_suffix: ${{ inputs.artifact_suffix }}
secrets: inherit
publish_model:
if: ${{ !inputs.bypass_push && !cancelled() }}
concurrency:
group: gitlab-push-${{ inputs.recompiled_dir }}
cancel-in-progress: false
needs: build_model
runs-on: ubuntu-latest
env:
RECOMPILED_DIR: recompiled${{ github.event.inputs.recompiled_dir }}
JSON_FILE: docs/docs/driving_models_v${{ github.event.inputs.json_version }}.json
steps:
- name: Set up SSH
uses: webfactory/ssh-agent@v0.9.0
@@ -96,66 +135,49 @@ jobs:
exit 1
fi
- name: Install dependencies
- name: Download artifact name file
uses: actions/download-artifact@v4
with:
name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
path: artifact_name
- name: Read artifact name
id: read-artifact-name
run: |
sudo apt-get update
sudo apt-get install -y jq gh
- name: Build model
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
START_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
gh workflow run sunnypilot-build-model.yaml \
--repo sunnypilot/sunnypilot \
--ref "${{ github.event.inputs.build_model_ref }}" \
-f upstream_branch="${{ github.event.inputs.upstream_branch }}" \
-f custom_name="${{ github.event.inputs.custom_name }}"
for i in {1..24}; do
RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --created ">$START_TIME" --limit=1 --json databaseId --jq '.[0].databaseId')
if [ -n "$RUN_ID" ]; then
break
fi
sleep 5
done
if [ -z "$RUN_ID" ]; then
echo "Could not find the triggered workflow run."
exit 1
fi
echo "Watching run ID: $RUN_ID"
gh run watch "$RUN_ID" --repo sunnypilot/sunnypilot
CONCLUSION=$(gh run view "$RUN_ID" --repo sunnypilot/sunnypilot --json conclusion --jq '.conclusion')
echo "Run concluded with: $CONCLUSION"
if [[ "$CONCLUSION" != "success" ]]; then
echo "Workflow run failed with conclusion: $CONCLUSION"
exit 1
fi
ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
- name: Download and extract model artifact
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
uses: actions/download-artifact@v4
with:
name: ${{ steps.read-artifact-name.outputs.artifact_name }}
path: output
- name: Remove unwanted files
run: |
ARTIFACT_DIR="gitlab_docs/models/$RECOMPILED_DIR"
RUN_ID=$(gh run list --repo sunnypilot/sunnypilot --workflow=sunnypilot-build-model.yaml --branch="${{ github.event.inputs.build_model_ref }}" --limit=1 --json databaseId --jq '.[0].databaseId')
ARTIFACT_NAME=$(gh api repos/sunnypilot/sunnypilot/actions/runs/$RUN_ID/artifacts --jq '.artifacts[] | select(.name | startswith("model-")) | .name')
echo "Downloading artifact: $ARTIFACT_NAME from run: $RUN_ID"
mkdir -p "$ARTIFACT_DIR/$ARTIFACT_NAME"
echo "Created directory: $ARTIFACT_DIR/$ARTIFACT_NAME"
gh run download "$RUN_ID" --repo sunnypilot/sunnypilot -n "$ARTIFACT_NAME" --dir "$ARTIFACT_DIR/$ARTIFACT_NAME"
echo "Downloaded artifact zip(s) to: $ARTIFACT_DIR/$ARTIFACT_NAME"
ZIP_PATH=$(find "$ARTIFACT_DIR/$ARTIFACT_NAME" -type f -name '*.zip' | head -n1)
if [ -n "$ZIP_PATH" ]; then
echo "Unzipping $ZIP_PATH to $ARTIFACT_DIR/$ARTIFACT_NAME"
unzip -o "$ZIP_PATH" -d "$ARTIFACT_DIR/$ARTIFACT_NAME"
rm -f "$ZIP_PATH"
echo "Unzipped and removed $ZIP_PATH"
else
echo "No zip file found in $ARTIFACT_DIR/$ARTIFACT_NAME"
fi
echo "Done processing $ARTIFACT_NAME"
find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
find output -type f -name 'dmonitoring_model.onnx' -delete
- name: Copy model artifact(s) to GitLab recompiled dir
env:
ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
run: |
ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
mkdir -p "$ARTIFACT_DIR"
for path in output/*; do
if [ "$(basename "$path")" = "artifact_name.txt" ]; then
continue
fi
name="$(basename "$path")"
if [ -d "$path" ]; then
mkdir -p "$ARTIFACT_DIR/$name"
cp -r "$path"/* "$ARTIFACT_DIR/$name/"
echo "Copied dir $name -> $ARTIFACT_DIR/$name"
else
cp "$path" "$ARTIFACT_DIR/"
echo "Copied file $name -> $ARTIFACT_DIR/"
fi
done
- name: Push recompiled dir to GitLab
env:
@@ -163,28 +185,36 @@ jobs:
run: |
cd gitlab_docs
git checkout main
git pull origin main
for d in models/"$RECOMPILED_DIR"/*/; do
git sparse-checkout add "$d"
done
git add models/"$RECOMPILED_DIR"
git config --global user.name "GitHub Action"
git config --global user.email "action@github.com"
git commit -m "Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
git push origin main
- run: |
cd docs
git pull origin gh-pages
- name: Run json_parser.py to update JSON
run: |
FOLDER="${{ github.event.inputs.model_folder }}"
FOLDER="${{ inputs.model_folder }}"
if [ "$FOLDER" = "Other" ]; then
FOLDER="${{ github.event.inputs.custom_model_folder }}"
FOLDER="${{ inputs.custom_model_folder }}"
fi
ARGS=""
[ -n "$FOLDER" ] && ARGS="$ARGS --model-folder \"$FOLDER\""
[ -n "${{ github.event.inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ github.event.inputs.generation }}\""
[ -n "${{ github.event.inputs.version }}" ] && ARGS="$ARGS --version \"${{ github.event.inputs.version }}\""
if [ "$FOLDER" != "None" ] && [ -n "$FOLDER" ]; then
ARGS="$ARGS --model-folder \"$FOLDER\""
fi
[ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
[ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
eval python3 docs/json_parser.py \
--json-path "$JSON_FILE" \
--recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
--sort-by-date \
$ARGS
- name: Push updated JSON to GitHub docs repo
.github/workflows/cereal_validation.yaml
+1 -2
View File
@@ -4,7 +4,6 @@ on:
push:
branches:
- master
- master-new
pull_request:
paths:
- 'cereal/**'
@@ -17,7 +16,7 @@ on:
type: string
concurrency:
group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
.github/workflows/compile-openpilot/action.yaml
+1 -1
View File
@@ -15,7 +15,7 @@ runs:
scons -j$(nproc) --cache-populate"
- name: Save scons cache
uses: actions/cache/save@v4
if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
if: github.ref == 'refs/heads/master'
with:
path: .ci_cache/scons_cache
key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
.github/workflows/jenkins-pr-trigger.yaml
+14
View File
@@ -9,6 +9,9 @@ jobs:
scan-comments:
runs-on: ubuntu-latest
if: ${{ github.event.issue.pull_request }}
permissions:
contents: write
issues: write
steps:
- name: Check for trigger phrase
id: check_comment
@@ -43,3 +46,14 @@ jobs:
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b tmp-jenkins-${{ github.event.issue.number }}
GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
- name: Delete trigger comment
if: steps.check_comment.outputs.result == 'true' && always()
uses: actions/github-script@v7
with:
script: |
await github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
});
.github/workflows/lfs-maintenance.yaml
+5 -5
View File
@@ -9,11 +9,11 @@ on:
- cron: '0 0 * * *' # Runs at 00:00 UTC every day
push:
branches:
- 'master-new'
- 'master'
pull_request:
branches:
- 'master-new'
workflow_dispatch: # enables manual triggering
- 'master'
workflow_dispatch: # enables manual triggering
inputs:
upstream_branch:
default: 'master'
@@ -30,7 +30,7 @@ jobs:
with:
repository: 'commaai/openpilot'
ref: ${{ inputs.upstream_branch }}
- name: LFS Fetch
run: |
git lfs fetch
@@ -48,7 +48,7 @@ jobs:
- name: Add GitLab public keys
run: |
ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
- name: Ensure branch
run: |
if git symbolic-ref -q HEAD >/dev/null; then
.github/workflows/prebuilt.yaml
+1 -1
View File
@@ -6,7 +6,7 @@ on:
env:
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh prebuilt
BUILD: release/ci/docker_build_sp.sh prebuilt
jobs:
build_prebuilt:
.github/workflows/release-drafter.yml
-1
View File
@@ -3,7 +3,6 @@ name: Release Drafter
on:
push:
branches:
- master-new
- master
tags:
- 'v*'
.github/workflows/release.yaml
+5 -5
View File
@@ -5,12 +5,12 @@ on:
workflow_dispatch:
jobs:
build_masterci:
name: build master-ci
build___nightly:
name: build __nightly
env:
ImageOS: ubuntu24
container:
image: ghcr.io/commaai/openpilot-base:latest
image: ghcr.io/sunnypilot/sunnypilot-base:latest
runs-on: ubuntu-latest
if: github.repository == 'sunnypilot/sunnypilot'
permissions:
@@ -27,7 +27,7 @@ jobs:
with:
ref: master
wait-interval: 30
running-workflow-name: 'build master-ci'
running-workflow-name: 'build __nightly'
repo-token: ${{ secrets.GITHUB_TOKEN }}
check-regexp: ^((?!.*(build prebuilt).*).)*$
- uses: actions/checkout@v4
@@ -38,5 +38,5 @@ jobs:
run: |
git config --global --add safe.directory '*'
git lfs pull
- name: Push master-ci
- name: Push __nightly
run: BRANCH=__nightly release/build_devel.sh
.github/workflows/repo-maintenance.yaml
+5 -5
View File
@@ -6,8 +6,8 @@ on:
workflow_dispatch:
env:
BASE_IMAGE: openpilot-base
BUILD: selfdrive/test/docker_build.sh base
BASE_IMAGE: sunnypilot-base
BUILD: release/ci/docker_build_sp.sh base
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
jobs:
@@ -28,7 +28,7 @@ jobs:
title: "[bot] Update translations"
body: "Automatic PR from repo-maintenance -> update_translations"
branch: "update-translations"
base: "master-new"
base: "master"
delete-branch: true
labels: bot
@@ -36,7 +36,7 @@ jobs:
name: package_updates
runs-on: ubuntu-latest
container:
image: ghcr.io/commaai/openpilot-base:latest
image: ghcr.io/sunnypilot/sunnypilot-base:latest
if: github.repository == 'sunnypilot/sunnypilot'
steps:
- uses: actions/checkout@v4
@@ -66,7 +66,7 @@ jobs:
commit-message: Update Python packages
title: '[bot] Update Python packages'
branch: auto-package-updates
base: master-new
base: master
delete-branch: true
body: 'Automatic PR from repo-maintenance -> package_updates'
labels: bot
.github/workflows/selfdrive_tests.yaml
+77 -115
View File
@@ -4,7 +4,6 @@ on:
push:
branches:
- master
- master-new
pull_request:
workflow_dispatch:
workflow_call:
@@ -15,17 +14,16 @@ on:
type: string
concurrency:
group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
PYTHONWARNINGS: error
BASE_IMAGE: openpilot-base
BASE_IMAGE: sunnypilot-base
AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh base
BUILD: release/ci/docker_build_sp.sh base
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
@@ -33,10 +31,13 @@ env:
jobs:
build_release:
if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
name: build release
runs-on:
- 'ubuntu-24.04'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
env:
STRIPPED_DIR: /tmp/releasepilot
steps:
@@ -66,17 +67,37 @@ jobs:
- name: Check submodules
if: github.repository == 'sunnypilot/sunnypilot'
timeout-minutes: 3
run: release/check-submodules.sh
run: |
if [ "${{ github.ref }}" != "refs/heads/master" ]; then
git fetch origin master:refs/remotes/origin/master
SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
if git ls-files --stage "$path" | grep -q "^160000"; then
echo "$path"
fi
done | tr '\n' ' ')
if [ -n "$SUBMODULE_PATHS" ]; then
echo "Changed submodule paths: $SUBMODULE_PATHS"
export SUBMODULE_PATHS="$SUBMODULE_PATHS"
export CHECK_PR_REFS=true
fi
fi
release/check-submodules.sh
build:
runs-on:
- 'ubuntu-24.04'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Setup docker push
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'sunnypilot/sunnypilot'
run: |
echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
$DOCKER_LOGIN
@@ -86,6 +107,7 @@ jobs:
build_mac:
name: build macOS
if: false # temp disable since homebrew install is getting stuck
runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
steps:
- uses: actions/checkout@v4
@@ -107,7 +129,7 @@ jobs:
PYTHONWARNINGS: default
- name: Save Homebrew cache
uses: actions/cache/save@v4
if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
if: github.ref == 'refs/heads/master'
with:
path: ~/Library/Caches/Homebrew
key: brew-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
@@ -124,15 +146,19 @@ jobs:
run: . .venv/bin/activate && scons -j$(nproc)
- name: Save scons cache
uses: actions/cache/save@v4
if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new')
if: github.ref == 'refs/heads/master'
with:
path: /tmp/scons_cache
key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
static_analysis:
name: static analysis
runs-on:
- 'ubuntu-latest'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
env:
PYTHONWARNINGS: default
steps:
@@ -146,19 +172,25 @@ jobs:
unit_tests:
name: unit tests
runs-on:
- 'ubuntu-24.04'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
id: setup-step
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Run unit tests
timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 999 }}
run: |
${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
$PYTEST --collect-only -m 'not slow' &> /dev/null && \
MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
./selfdrive/ui/tests/create_test_translations.sh && \
QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
@@ -166,14 +198,19 @@ jobs:
process_replay:
name: process replay
if: github.repository == 'commaai/openpilot' # disable process_replay for forks
runs-on:
- 'ubuntu-24.04'
if: false # disable process_replay for forks
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
id: setup-step
- name: Cache test routes
id: dependency-cache
uses: actions/cache@v4
@@ -184,7 +221,7 @@ jobs:
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
run: |
${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache"
@@ -209,106 +246,26 @@ jobs:
${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
chmod -R 777 /tmp/comma_download_cache"
test_cars:
name: cars
runs-on:
- 'ubuntu-24.04'
strategy:
fail-fast: false
matrix:
job: [0, 1, 2, 3]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Cache test routes
id: routes-cache
uses: actions/cache@v4
with:
path: .ci_cache/comma_download_cache
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Test car models
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 6 }}
run: |
${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
chmod -R 777 /tmp/comma_download_cache"
env:
NUM_JOBS: 4
JOB_ID: ${{ matrix.job }}
car_docs_diff:
name: PR comments
runs-on: ubuntu-latest
#if: github.event_name == 'pull_request'
if: false # TODO: run this in opendbc?
steps:
- uses: actions/checkout@v4
with:
submodules: true
ref: ${{ github.event.pull_request.base.ref }}
- run: git lfs pull
- uses: ./.github/workflows/setup-with-retry
- name: Get base car info
run: |
${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
sudo chown -R $USER:$USER ${{ github.workspace }}
- uses: actions/checkout@v4
with:
submodules: true
path: current
- run: cd current && git lfs pull
- name: Save car docs diff
id: save_diff
run: |
cd current
${{ env.RUN }} "scons -j$(nproc)"
output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
output="${output//$'\n'/'%0A'}"
echo "::set-output name=diff::$output"
- name: Find comment
if: ${{ env.AZURE_TOKEN != '' }}
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: This PR makes changes to
- name: Update comment
if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: "${{ steps.save_diff.outputs.diff }}"
edit-mode: replace
- name: Delete comment
if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
uses: actions/github-script@v7
with:
script: |
github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: ${{ steps.fc.outputs.comment-id }}
})
simulator_driving:
name: simulator driving
runs-on:
- 'ubuntu-24.04'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
id: setup-step
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Driving test
timeout-minutes: 1
timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
run: |
${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
source selfdrive/test/setup_vsound.sh && \
@@ -317,8 +274,13 @@ jobs:
create_ui_report:
# This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
name: Create UI Report
runs-on:
- 'ubuntu-24.04'
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
if: false # FIXME: FrameReader is broken on CI runners
steps:
- uses: actions/checkout@v4
with:
@@ -342,5 +304,5 @@ jobs:
- name: Upload Test Report
uses: actions/upload-artifact@v4
with:
name: ${{ env.REPORT_NAME }}
name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
path: selfdrive/ui/tests/test_ui/report_1/screenshots
.github/workflows/setup-with-retry/action.yaml
+15
View File
@@ -10,9 +10,17 @@ inputs:
required: false
default: 30
outputs:
duration:
description: 'Duration of the setup process in seconds'
value: ${{ steps.get_duration.outputs.duration }}
runs:
using: "composite"
steps:
- id: start_time
shell: bash
run: echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
- id: setup1
uses: ./.github/workflows/setup
continue-on-error: true
@@ -35,3 +43,10 @@ runs:
uses: ./.github/workflows/setup
with:
is_retried: true
- id: get_duration
shell: bash
run: |
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
echo "Total duration: $DURATION seconds"
echo "duration=$DURATION" >> $GITHUB_OUTPUT
.github/workflows/sunnypilot-build-model.yaml
+62 -12
View File
@@ -9,6 +9,27 @@ env:
MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
on:
workflow_call:
inputs:
upstream_branch:
description: 'Upstream branch to build from'
required: true
default: 'master'
type: string
custom_name:
description: 'Custom name for the model (no date, only name)'
required: false
type: string
is_20hz:
description: 'Is this a 20Hz model'
required: false
type: boolean
default: true
artifact_suffix:
description: 'Suffix for artifact name'
required: false
type: string
default: ''
workflow_dispatch:
inputs:
upstream_branch:
@@ -32,34 +53,53 @@ run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from
jobs:
get_model:
runs-on: ubuntu-latest
env:
REF: ${{ inputs.upstream_branch }}
outputs:
model_date: ${{ steps.commit-date.outputs.model_date }}
steps:
- uses: actions/checkout@v4
# Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
# and fallback to sunnypilot if the ref checkout fails.
- name: Checkout commaai/openpilot
id: checkout_upstream
continue-on-error: true
uses: actions/checkout@v4
with:
repository: ${{ env.UPSTREAM_REPO }}
ref: ${{ github.event.inputs.upstream_branch }}
repository: commaai/openpilot
ref: ${{ inputs.upstream_branch }}
submodules: recursive
path: openpilot
- name: Fallback to sunnypilot/sunnypilot
if: steps.checkout_upstream.outcome == 'failure'
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot
ref: ${{ inputs.upstream_branch }}
submodules: recursive
path: openpilot
- name: Get commit date
id: commit-date
run: |
# Get the commit date in YYYY-MM-DD format
cd ${{ github.workspace }}/openpilot
commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
cat $GITHUB_OUTPUT
- run: git lfs pull
- run: |
cd ${{ github.workspace }}/openpilot
git lfs pull
- name: 'Upload Artifact'
uses: actions/upload-artifact@v4
with:
name: models
path: ${{ github.workspace }}/selfdrive/modeld/models/*.onnx
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
build_model:
runs-on: self-hosted
runs-on: [self-hosted, tici]
needs: get_model
env:
MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
REF: ${{ inputs.upstream_branch }}
steps:
- uses: actions/checkout@v4
with:
@@ -71,7 +111,7 @@ jobs:
with:
path: ${{env.SCONS_CACHE_DIR}}
key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
# Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
# Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
# for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
restore-keys: |
scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
@@ -114,7 +154,7 @@ jobs:
- name: Download model artifacts
uses: actions/download-artifact@v4
with:
name: models
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ github.workspace }}/selfdrive/modeld/models
- name: Build Model
@@ -157,12 +197,22 @@ jobs:
--upstream-branch "${{ inputs.upstream_branch }}" \
${{ inputs.is_20hz && '--is-20hz' || '' }}
- name: Write artifact name to file
run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
- name: Upload Build Artifacts
id: upload-artifact
uses: actions/upload-artifact@v4
with:
name: model-${{ env.MODEL_NAME }}-${{ github.run_number }}
name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
path: ${{ env.OUTPUT_DIR }}
- name: Upload artifact name file
uses: actions/upload-artifact@v4
with:
name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
- name: Re-enable powersave
if: always()
run: |
.github/workflows/sunnypilot-build-prebuilt.yaml
+23 -24
View File
@@ -8,7 +8,7 @@ env:
PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
# Branch configurations
STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master-new' }} # vars are set on repo settings.
STAGING_C3_SOURCE_BRANCH: ${{ vars.STAGING_C3_SOURCE_BRANCH || 'master' }} # vars are set on repo settings.
DEV_C3_SOURCE_BRANCH: ${{ vars.DEV_C3_SOURCE_BRANCH || 'master-dev-c3-new' }} # vars are set on repo settings.
# Target branch configurations
@@ -21,7 +21,7 @@ env:
on:
push:
branches: [ master, master-new, master-dev-c3-new ]
branches: [ master, master-dev-c3-new ]
tags: [ '*' ]
pull_request_target:
types: [ labeled ]
@@ -50,7 +50,7 @@ jobs:
concurrency:
group: build-${{ github.head_ref || github.ref_name }}
cancel-in-progress: false
runs-on: self-hosted
runs-on: [self-hosted, tici]
outputs:
new_branch: ${{ steps.set-env.outputs.new_branch }}
version: ${{ steps.set-env.outputs.version }}
@@ -80,10 +80,10 @@ jobs:
- name: Set Feature Branch Prebuilt Configuration
id: set_feature_configuration
if: (
!(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
!(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
!(startsWith(github.ref, 'refs/tags/'))
)
!(env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH) &&
!(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH) &&
!(startsWith(github.ref, 'refs/tags/'))
)
run: |
echo "NEW_BRANCH=${{ env.SOURCE_BRANCH }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-prebuilt" >> $GITHUB_ENV
echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
@@ -91,9 +91,9 @@ jobs:
- name: Set dev-c3-new prebuilt Configuration
id: set_dev_configuration
if: (
steps.set_feature_configuration.outcome == 'skipped' &&
env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
)
steps.set_feature_configuration.outcome == 'skipped' &&
env.SOURCE_BRANCH == env.DEV_C3_SOURCE_BRANCH
)
run: |
echo "NEW_BRANCH=${{ env.DEV_TARGET_BRANCH }}" >> $GITHUB_ENV
echo "VERSION=$(date '+%Y.%m.%d')-${{ github.run_number }}" >> $GITHUB_ENV
@@ -102,11 +102,11 @@ jobs:
- name: Set staging-c3-new prebuilt Configuration
id: set_staging_configuration
if: (
steps.set_feature_configuration.outcome == 'skipped' &&
!contains(github.event_name, 'pull_request') &&
steps.set_dev_configuration.outcome == 'skipped' &&
(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
)
steps.set_feature_configuration.outcome == 'skipped' &&
!contains(github.event_name, 'pull_request') &&
steps.set_dev_configuration.outcome == 'skipped' &&
(env.SOURCE_BRANCH == env.STAGING_C3_SOURCE_BRANCH)
)
run: |
echo "NEW_BRANCH=${{ env.STAGING_TARGET_BRANCH }}" >> $GITHUB_ENV
echo "EXTRA_VERSION_IDENTIFIER=staging" >> $GITHUB_ENV
@@ -115,11 +115,11 @@ jobs:
- name: Set release-c3-new prebuilt Configuration
id: set_tag_configuration
if: (
steps.set_feature_configuration.outcome == 'skipped' &&
!contains(github.event_name, 'pull_request') &&
steps.set_staging_configuration.outcome == 'skipped' &&
startsWith(github.ref, 'refs/tags/')
)
steps.set_feature_configuration.outcome == 'skipped' &&
!contains(github.event_name, 'pull_request') &&
steps.set_staging_configuration.outcome == 'skipped' &&
startsWith(github.ref, 'refs/tags/')
)
run: |
echo "NEW_BRANCH=${{ env.RELEASE_TARGET_BRANCH }}" >> $GITHUB_ENV
echo "EXTRA_VERSION_IDENTIFIER=release" >> $GITHUB_ENV
@@ -227,7 +227,6 @@ jobs:
run: |
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
publish:
concurrency:
group: publish-${{ github.head_ref || github.ref_name }}
@@ -268,7 +267,7 @@ jobs:
"${{ needs.build.outputs.version }}" \
"https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
"-${{ needs.build.outputs.extra_version_identifier }}"
echo ""
echo "---- ℹ️ To update the list of branches that auto deploy prebuilts -----"
echo ""
@@ -299,7 +298,7 @@ jobs:
export extra_version_identifier=${{ needs.build.outputs.extra_version_identifier || github.run_number}}
echo ${TEMPLATE} | envsubst | jq -c '.' | tee payload.json
curl -X POST -H "Content-Type: application/json" -d @payload.json $DISCORD_WEBHOOK
echo ""
echo "---- ℹ️ To update the list of branches that notify to dev-feedback -----"
echo ""
@@ -307,7 +306,7 @@ jobs:
echo "2. Current value: ${{ vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES }}"
echo "3. Update as needed (JSON array with no spaces)"
shell: alpine.sh {0}
manage-pr-labels:
name: Remove prebuilt label
runs-on: ubuntu-latest
.github/workflows/sunnypilot-master-dev-c3-prep.yaml
+13 -14
View File
@@ -1,7 +1,7 @@
name: Build dev-c3-new
env:
DEFAULT_SOURCE_BRANCH: "master-new"
DEFAULT_SOURCE_BRANCH: "master"
DEFAULT_TARGET_BRANCH: "master-dev-c3-new"
PR_LABEL: "dev-c3"
LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
@@ -11,18 +11,16 @@ on:
push:
branches:
- master
- master-new
pull_request_target:
types: [ labeled ]
branches:
- 'master'
- 'master-new'
types: [ labeled ]
branches:
- 'master'
workflow_dispatch:
inputs:
source_branch:
description: 'Source branch to reset from'
required: true
default: 'master-new'
default: 'master'
type: string
target_branch:
description: 'Target branch to reset and squash into'
@@ -43,10 +41,10 @@ jobs:
reset-and-squash:
runs-on: ubuntu-latest
if: (
(github.event_name == 'workflow_dispatch')
|| (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
|| (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
)
(github.event_name == 'workflow_dispatch')
|| (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
|| (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
)
steps:
- uses: actions/checkout@v4
with:
@@ -56,9 +54,9 @@ jobs:
- name: Wait for Tests
uses: ./.github/workflows/wait-for-action # Path to where you place the action
if: (
(github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
|| (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
)
(github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
|| (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == 'dev-c3' || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, 'dev-c3'))))
)
with:
workflow: selfdrive_tests.yaml # The workflow file to monitor
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -153,6 +151,7 @@ jobs:
}
}' -F label="is:pr is:open label:${PR_LABEL} draft:false sort:created-asc")
PR_LIST=${PR_LIST//\'/}
echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/ui_preview.yaml
+3 -5
View File
@@ -3,20 +3,18 @@ on:
push:
branches:
- master
- master-new
pull_request_target:
types: [assigned, opened, synchronize, reopened, edited]
branches:
- 'master'
- 'master-new'
paths:
- 'selfdrive/ui/**'
workflow_dispatch:
env:
UI_JOB_NAME: "Create UI Report"
REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
jobs:
@@ -66,7 +64,7 @@ jobs:
ref: openpilot_master_ui
- name: Saving new master ui
if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
working-directory: ${{ github.workspace }}/master_ui
run: |
git checkout --orphan=new_master_ui