mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-08-04 22:15:42 +08:00
Add CI/CD pipeline and Docker support
This commit is contained in:
DevTekVE
+163
@@ -0,0 +1,163 @@
|
||||
variables:
|
||||
BUILD_DIR: "/data/openpilot"
|
||||
OUTPUT_DIR: "${CI_PROJECT_DIR}/output"
|
||||
CI_DIR: "${CI_PROJECT_DIR}/release/ci"
|
||||
VERSION: "unknown"
|
||||
GIT_SUBMODULE_STRATEGY: recursive
|
||||
GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
|
||||
DEV_C3_SOURCE_BRANCH: "master-dev-c3"
|
||||
|
||||
stages:
|
||||
- build
|
||||
- commit_changes
|
||||
- publish
|
||||
|
||||
default:
|
||||
tags:
|
||||
- sunnypilot
|
||||
- x86
|
||||
|
||||
.default_before_script: &default_before_script
|
||||
- 'export VERSION=$(date "+%Y.%m.%d")-${CI_PIPELINE_IID}'
|
||||
- 'mkdir -p "${BUILD_DIR}/"'
|
||||
- 'git config --global user.email "gitlab@pipeline.com"'
|
||||
- 'git config --global user.name "Gitlab Pipeline"'
|
||||
|
||||
|
||||
workflow: # If running on any branch other than main, use the `aws-datacontracts-dev` account; otherwise use `aws-datacontracts`
|
||||
rules:
|
||||
# We are an MR, but it's a draft, we won't proceed with anything.
|
||||
- if: '$CI_MERGE_REQUEST_TITLE =~ /^wip:/i || $CI_MERGE_REQUEST_TITLE =~ /^draft:/i'
|
||||
when: never
|
||||
# We are a merge request
|
||||
- if: $CI_MERGE_REQUEST_IID #|| $CI_COMMIT_REF_NAME == "gitlab-pipelines" # TBD once merged
|
||||
variables:
|
||||
NEW_BRANCH: ${CI_COMMIT_REF_NAME}-prebuilt
|
||||
when: always
|
||||
|
||||
# Below are the rules when a commit is done (code has been added to the branch)
|
||||
# Commit to master-dev-c3
|
||||
- if: $CI_COMMIT_REF_NAME == $DEV_C3_SOURCE_BRANCH
|
||||
variables:
|
||||
NEW_BRANCH: "dev-c3"
|
||||
when: always
|
||||
#commit made to main (master)
|
||||
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
|
||||
variables:
|
||||
NEW_BRANCH: "staging-c3"
|
||||
when: always
|
||||
# if tag
|
||||
- if: $CI_COMMIT_TAG
|
||||
variables:
|
||||
NEW_BRANCH: "release-c3"
|
||||
- when: always
|
||||
|
||||
|
||||
build:
|
||||
stage: build
|
||||
cache:
|
||||
key: "${CI_COMMIT_REF_SLUG}"
|
||||
paths:
|
||||
- "${CI_DIR}/scons_cache"
|
||||
before_script:
|
||||
- *default_before_script
|
||||
- "echo Starting build stage..."
|
||||
- "echo BUILD_DIR: ${BUILD_DIR}"
|
||||
- "echo CI_DIR: ${CI_DIR}"
|
||||
- "echo VERSION: ${VERSION}"
|
||||
- "echo CI_COMMIT_REF_NAME: ${CI_COMMIT_REF_NAME}"
|
||||
- git config --global --add safe.directory ${CI_PROJECT_DIR}
|
||||
script:
|
||||
- export PYTHONPATH="$BUILD_DIR"
|
||||
- "echo Calling to build [${CI_DIR}/build.sh ${CI_PROJECT_DIR} ${BUILD_DIR} ${VERSION}]"
|
||||
- "echo Building Panda..."
|
||||
- scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache ${CI_PROJECT_DIR}/panda
|
||||
- mkdir -p ${BUILD_DIR}
|
||||
- ls -la ${BUILD_DIR}
|
||||
- "echo Building Rest..."
|
||||
- cp -pR --parents $(cat release/files_common release/files_tici) $BUILD_DIR/
|
||||
- cd $BUILD_DIR
|
||||
- sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py # comment panda jungle when prebuilt
|
||||
- scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache
|
||||
|
||||
# Cleanup
|
||||
- echo "Doing cleanup for prebuilt\n\n"
|
||||
- echo "Deleting specified files..."
|
||||
- find ${BUILD_DIR} \( -name '*.a' -o -name '*.o' -o -name '*.os' -o -name '*.pyc' -o -name 'moc_*' -o -name '*.cc' -o -name "Jenkinsfile" -o -name "supercombo.onnx" -o -name ".sconsign.dblite" \) -print -delete
|
||||
|
||||
- echo "Deleting Headers from selfdrive/ui path..."
|
||||
- find ${BUILD_DIR}/selfdrive/ui -type f -name '*.h' -print -delete
|
||||
|
||||
- echo "Deleting the following folders and subdirectories..."
|
||||
- find "${BUILD_DIR}" -type d \( -path "*panda/board" -o -path "*panda/certs" -o -path "*panda/crypto" -o -path "*release" -o -path "*.github" -o -path "*selfdrive/ui/replay" -o -path "*__pycache__" \) -print -exec rm -rf {} +
|
||||
|
||||
# Move back signed panda fw
|
||||
- mkdir -p ${BUILD_DIR}/panda/board/obj
|
||||
- cp ${CI_PROJECT_DIR}/panda/board/obj/panda.bin.signed ${BUILD_DIR}/panda/board/obj/panda.bin.signed
|
||||
- cp ${CI_PROJECT_DIR}/panda/board/obj/panda_h7.bin.signed ${BUILD_DIR}/panda/board/obj/panda_h7.bin.signed
|
||||
- cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda.bin
|
||||
- cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda_h7.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda_h7.bin
|
||||
- touch ${BUILD_DIR}/prebuilt
|
||||
- mkdir -p ${OUTPUT_DIR}
|
||||
- shopt -s dotglob && mv ${BUILD_DIR}/* ${OUTPUT_DIR}
|
||||
artifacts:
|
||||
paths:
|
||||
- ${OUTPUT_DIR}/
|
||||
tags: [ 'sunnypilot', 'tici' ]
|
||||
rules:
|
||||
- if: $CI_MERGE_REQUEST_IID
|
||||
when: manual
|
||||
- if: $NEW_BRANCH
|
||||
when: always
|
||||
|
||||
|
||||
commit changes for public branch:
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
stage: commit_changes
|
||||
dependencies:
|
||||
- build
|
||||
before_script:
|
||||
- 'apk update && apk upgrade'
|
||||
- 'apk add git bash openssh'
|
||||
- 'eval $(ssh-agent -s)'
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
||||
- 'mkdir -p ~/.ssh/'
|
||||
- 'chmod 700 ~/.ssh'
|
||||
- 'ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts' # Adding gitlab to trusted
|
||||
- 'chmod 644 ~/.ssh/known_hosts'
|
||||
- *default_before_script
|
||||
script:
|
||||
- GIT_ORIGIN=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g')
|
||||
- echo "${GIT_ORIGIN}"
|
||||
- echo "Calling to publish [${CI_DIR}/publish.sh ${CI_PROJECT_DIR} ${OUTPUT_DIR} ${NEW_BRANCH} ${VERSION} ${GIT_ORIGIN}]"
|
||||
- git config --global --add safe.directory ${OUTPUT_DIR}
|
||||
- $CI_DIR/publish.sh "${CI_PROJECT_DIR}" "${OUTPUT_DIR}" "${NEW_BRANCH}" "${VERSION}" "${GIT_ORIGIN}"
|
||||
rules:
|
||||
- if: $CI_MERGE_REQUEST_IID && $NEW_BRANCH
|
||||
when: on_success
|
||||
- if: $CI_MERGE_REQUEST_IID == "null" && $NEW_BRANCH
|
||||
when: on_success
|
||||
- when: never
|
||||
|
||||
publish to github:
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: normal
|
||||
stage: publish
|
||||
dependencies:
|
||||
- build
|
||||
before_script:
|
||||
- 'apk update && apk upgrade'
|
||||
- 'apk add git bash openssh'
|
||||
- 'eval $(ssh-agent -s)'
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
||||
- 'mkdir -p ~/.ssh/'
|
||||
- 'chmod 700 ~/.ssh'
|
||||
- 'ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts' # Adding gitlab to trusted
|
||||
- 'chmod 644 ~/.ssh/known_hosts'
|
||||
- *default_before_script
|
||||
script:
|
||||
- echo "This part is not yet implemented intentionally, just an example"
|
||||
rules:
|
||||
- if: $NEW_BRANCH
|
||||
when: manual
|
||||
+2
-1
@@ -230,7 +230,8 @@ if GetOption('compile_db'):
|
||||
env.CompilationDatabase('compile_commands.json')
|
||||
|
||||
# Setup cache dir
|
||||
cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
|
||||
default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
|
||||
cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
|
||||
CacheDir(cache_dir)
|
||||
Clean(["."], cache_dir)
|
||||
|
||||
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON
|
||||
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# set -e
|
||||
|
||||
# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
# cd $DIR
|
||||
|
||||
# # Take parameters as arguments
|
||||
# SOURCE_DIR=$1
|
||||
# OUTPUT_DIR=$2
|
||||
|
||||
# # Check parameters
|
||||
# if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then
|
||||
# echo "Error: No source or output directory provided."
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# if [ -z "$TARGETPLATFORM" ]; then
|
||||
# # Detect the platform (e.g., x86_64, aarch64, etc.)
|
||||
# platform=$(uname -m)
|
||||
|
||||
# # Map to Docker's platform syntax
|
||||
# if [ "$platform" = "x86_64" ]; then
|
||||
# TARGETPLATFORM="linux/amd64"
|
||||
# elif [ "$platform" = "aarch64" ]; then
|
||||
# TARGETPLATFORM="linux/arm64"
|
||||
# # Add here more elif statements for other architectures if needed.
|
||||
# else
|
||||
# echo "Unknown platform: $platform"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# export TARGETPLATFORM
|
||||
# fi
|
||||
|
||||
# echo "TARGETPLATFORM IS: [${TARGETPLATFORM}]"
|
||||
|
||||
# # Let's bring the submodules!
|
||||
# git submodule update --init --recursive
|
||||
|
||||
# # Build
|
||||
# docker compose -f ${SOURCE_DIR}/docker-compose.yml build
|
||||
# docker compose -f ${SOURCE_DIR}/docker-compose.yml up
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON
|
||||
|
||||
# #!/usr/bin/env bash
|
||||
# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
# cd $DIR
|
||||
|
||||
# DEV_BRANCH=$1
|
||||
# GIT_ORIGIN=$2
|
||||
# if [ -z "$DEV_BRANCH" ]; then
|
||||
# DEV_BRANCH="dev-c3-test"
|
||||
# fi
|
||||
|
||||
# if [ -z "$GIT_ORIGIN" ]; then
|
||||
# GIT_ORIGIN="git@github.com:devtekve/openpilot-og.git" #This is a default that should be changed
|
||||
# echo "No GIT_ORIGIN provided, we will use the default [${GIT_ORIGIN}]"
|
||||
# fi
|
||||
|
||||
# VERSION=$(date '+%Y.%m.%d')
|
||||
# SOURCE_DIR="$(git rev-parse --show-toplevel)"
|
||||
# OUTPUT_DIR=${SOURCE_DIR}/output
|
||||
|
||||
|
||||
# echo "Calling to build [${DIR}/build.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${VERSION}]"
|
||||
# $DIR/build.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${VERSION}"
|
||||
|
||||
# echo "Calling to publish [${DIR}/publish.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${DEV_BRANCH} ${VERSION} ${GIT_ORIGIN}]"
|
||||
# $DIR/publish.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${DEV_BRANCH}" "${VERSION}" "${GIT_ORIGIN}"
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
|
||||
# We need RW for the install process
|
||||
sudo mount -o remount rw /
|
||||
|
||||
# Ensure filesystem is remounted as read-only on script exit
|
||||
trap "sudo mount -o remount ro /" EXIT
|
||||
|
||||
# Define directories
|
||||
BASE_DIR="/data/gitlab"
|
||||
BIN_DIR="$BASE_DIR/bin"
|
||||
CONFIG_DIR="$BASE_DIR"
|
||||
BUILDS_DIR="$BASE_DIR/builds"
|
||||
OPENPILOT_DIR="$BASE_DIR/openpilot"
|
||||
LOGS_DIR="$BASE_DIR/logs"
|
||||
CACHE_DIR="$BASE_DIR/cache"
|
||||
GITLAB_RUNNER_USERNAME="gitlab-runner"
|
||||
GROUPS_NEEDED="comma,gpu,gpio"
|
||||
|
||||
# Create necessary directories
|
||||
sudo mkdir -p "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR"
|
||||
|
||||
# Download the GitLab Runner binary
|
||||
sudo curl -L --output "$BIN_DIR/gitlab-runner" "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64"
|
||||
|
||||
# Give it permission to execute
|
||||
sudo chmod +x "$BIN_DIR/gitlab-runner"
|
||||
|
||||
# Create a GitLab Runner user
|
||||
sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${BASE_DIR} ${GITLAB_RUNNER_USERNAME} --shell /bin/bash -G ${GROUPS_NEEDED} || sudo usermod -aG ${GROUPS_NEEDED} gitlab-runner
|
||||
|
||||
# Clean bash_logout as it break gitlab pipelines
|
||||
sudo truncate -s 0 ${BASE_DIR}/.bash_logout
|
||||
|
||||
# Create a configuration file
|
||||
cat <<EOL | sudo tee "$CONFIG_DIR/config.toml"
|
||||
[[runners]]
|
||||
name = "tici"
|
||||
url = "https://gitlab.com/"
|
||||
token = "$1"
|
||||
executor = "shell"
|
||||
builds_dir = "$BUILDS_DIR"
|
||||
[runners.custom_build_dir]
|
||||
[runners.docker]
|
||||
volumes = ["$CACHE_DIR:/cache"]
|
||||
[runners.cache]
|
||||
MaxUploadedArchiveSize = 0
|
||||
[runners.custom]
|
||||
config_exec = "$LOGS_DIR"
|
||||
EOL
|
||||
|
||||
# Set permissions
|
||||
sudo chown -R ${GITLAB_RUNNER_USERNAME}:comma "$BASE_DIR"
|
||||
sudo chmod g+rwx "$BASE_DIR"
|
||||
sudo chmod g+s "$BASE_DIR"
|
||||
|
||||
|
||||
# Create a systemd service file for gitlab-runner
|
||||
cat <<EOL | sudo tee /etc/systemd/system/gitlab-runner.service
|
||||
[Unit]
|
||||
Description=GitLab Runner
|
||||
After=syslog.target network.target
|
||||
ConditionFileIsExecutable=$BIN_DIR/gitlab-runner
|
||||
|
||||
[Service]
|
||||
StartLimitInterval=5
|
||||
StartLimitBurst=10
|
||||
ExecStart=/usr/bin/unshare -m -- sh -c 'mount --bind $OPENPILOT_DIR /data/openpilot && exec $BIN_DIR/gitlab-runner "run" "--working-directory" "$BUILDS_DIR" "--config" "$CONFIG_DIR/config.toml" "--service" "gitlab-runner" "--syslog" "--user" "${GITLAB_RUNNER_USERNAME}"'
|
||||
|
||||
Restart=always
|
||||
RestartSec=120
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOL
|
||||
|
||||
# Reload systemd and start gitlab-runner
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl disable gitlab-runner # Intentionally, making sure the service is NOT enabled on boot.
|
||||
sudo systemctl start gitlab-runner
|
||||
Executable
+72
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
cd $DIR
|
||||
|
||||
# Take parameters as arguments
|
||||
SOURCE_DIR=$1
|
||||
OUTPUT_DIR=$2
|
||||
DEV_BRANCH=$3
|
||||
VERSION=$4
|
||||
GIT_ORIGIN=$5
|
||||
|
||||
# Check parameters
|
||||
if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then
|
||||
echo "Error: No source or output directory provided."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$DEV_BRANCH" ] || [ -z "$VERSION" ]; then
|
||||
echo "Error: No dev branch or version provided."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$GIT_ORIGIN" ]; then
|
||||
echo "Error: No GIT_ORIGIN provided"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# "Tagging"
|
||||
echo "#define COMMA_VERSION \"$VERSION-dev\"" > ${OUTPUT_DIR}/common/version.h
|
||||
|
||||
## set git identity
|
||||
#source $DIR/identity.sh
|
||||
#export GIT_SSH_COMMAND="ssh -i /data/gitkey"
|
||||
|
||||
echo "[-] Setting up repo T=$SECONDS"
|
||||
cd $OUTPUT_DIR
|
||||
git init
|
||||
|
||||
# set git username/password
|
||||
#source /data/identity.sh
|
||||
|
||||
git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .git doesn't exist or not allowed to delete
|
||||
git remote remove origin || true # ensure cleanup
|
||||
git remote add origin $GIT_ORIGIN
|
||||
#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out
|
||||
git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH)
|
||||
|
||||
echo "[-] committing version $VERSION T=$SECONDS"
|
||||
git add -f .
|
||||
git commit -a -m "sunnypilot v$VERSION release"
|
||||
git branch --set-upstream-to=origin/$DEV_BRANCH
|
||||
|
||||
# include source commit hash and build date in commit
|
||||
GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
|
||||
DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
|
||||
SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
|
||||
|
||||
# Add built files to git
|
||||
git add -f .
|
||||
git commit --amend -m "sunnypilot v$VERSION
|
||||
version: sunnypilot v$SP_VERSION release
|
||||
date: $DATETIME
|
||||
master commit: $GIT_HASH
|
||||
"
|
||||
git branch -m $DEV_BRANCH
|
||||
|
||||
# Push!
|
||||
echo "[-] pushing T=$SECONDS"
|
||||
git push -f origin $DEV_BRANCH
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Define the service name
|
||||
SERVICE_NAME="gitlab-runner"
|
||||
|
||||
# Function to control the service
|
||||
control_service() {
|
||||
local action=$1 # Store the function argument in a local variable
|
||||
sudo systemctl $action ${SERVICE_NAME}
|
||||
}
|
||||
|
||||
# Check for required argument
|
||||
if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then
|
||||
echo "Usage: $0 {start|stop}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Store the script argument in a descriptive variable
|
||||
ACTION=$1
|
||||
|
||||
# Trap EXIT signal (Ctrl+C) and stop the service
|
||||
trap 'control_service stop' SIGINT SIGKILL EXIT
|
||||
|
||||
# Enter the main loop
|
||||
while true; do
|
||||
control_service $ACTION # Call the function with the specified action
|
||||
sleep 1 # Pause before the next iteration
|
||||
done
|
||||
@@ -95,4 +95,11 @@ procs = [
|
||||
PythonProcess("webjoystick", "tools.bodyteleop.web", notcar),
|
||||
]
|
||||
|
||||
if os.path.exists("./gitlab_runner.sh") and True: # Of course and True is always true. Placeholder for a param :D
|
||||
# Only devs!
|
||||
procs += [
|
||||
NativeProcess("gitlab_runner_start", "selfdrive/manager", ["./gitlab_runner.sh", "start"], only_offroad, sigkill=False),
|
||||
NativeProcess("gitlab_runner_stop", "selfdrive/manager", ["./gitlab_runner.sh", "stop"], only_onroad, sigkill=False)
|
||||
]
|
||||
|
||||
managed_processes = {p.name: p for p in procs}
|
||||
|
||||
Reference in New Issue
Block a user