diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..78dad2758b --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,163 @@ +variables: + BUILD_DIR: "/data/openpilot" + OUTPUT_DIR: "${CI_PROJECT_DIR}/output" + CI_DIR: "${CI_PROJECT_DIR}/release/ci" + VERSION: "unknown" + GIT_SUBMODULE_STRATEGY: recursive + GIT_DISCOVERY_ACROSS_FILESYSTEM: 1 + DEV_C3_SOURCE_BRANCH: "master-dev-c3" + +stages: + - build + - commit_changes + - publish + +default: + tags: + - sunnypilot + - x86 + +.default_before_script: &default_before_script + - 'export VERSION=$(date "+%Y.%m.%d")-${CI_PIPELINE_IID}' + - 'mkdir -p "${BUILD_DIR}/"' + - 'git config --global user.email "gitlab@pipeline.com"' + - 'git config --global user.name "Gitlab Pipeline"' + + +workflow: # If running on any branch other than main, use the `aws-datacontracts-dev` account; otherwise use `aws-datacontracts` + rules: + # We are an MR, but it's a draft, we won't proceed with anything. + - if: '$CI_MERGE_REQUEST_TITLE =~ /^wip:/i || $CI_MERGE_REQUEST_TITLE =~ /^draft:/i' + when: never + # We are a merge request + - if: $CI_MERGE_REQUEST_IID #|| $CI_COMMIT_REF_NAME == "gitlab-pipelines" # TBD once merged + variables: + NEW_BRANCH: ${CI_COMMIT_REF_NAME}-prebuilt + when: always + + # Below are the rules when a commit is done (code has been added to the branch) + # Commit to master-dev-c3 + - if: $CI_COMMIT_REF_NAME == $DEV_C3_SOURCE_BRANCH + variables: + NEW_BRANCH: "dev-c3" + when: always + #commit made to main (master) + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + variables: + NEW_BRANCH: "staging-c3" + when: always + # if tag + - if: $CI_COMMIT_TAG + variables: + NEW_BRANCH: "release-c3" + - when: always + + +build: + stage: build + cache: + key: "${CI_COMMIT_REF_SLUG}" + paths: + - "${CI_DIR}/scons_cache" + before_script: + - *default_before_script + - "echo Starting build stage..." + - "echo BUILD_DIR: ${BUILD_DIR}" + - "echo CI_DIR: ${CI_DIR}" + - "echo VERSION: ${VERSION}" + - "echo CI_COMMIT_REF_NAME: ${CI_COMMIT_REF_NAME}" + - git config --global --add safe.directory ${CI_PROJECT_DIR} + script: + - export PYTHONPATH="$BUILD_DIR" + - "echo Calling to build [${CI_DIR}/build.sh ${CI_PROJECT_DIR} ${BUILD_DIR} ${VERSION}]" + - "echo Building Panda..." + - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache ${CI_PROJECT_DIR}/panda + - mkdir -p ${BUILD_DIR} + - ls -la ${BUILD_DIR} + - "echo Building Rest..." + - cp -pR --parents $(cat release/files_common release/files_tici) $BUILD_DIR/ + - cd $BUILD_DIR + - sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py # comment panda jungle when prebuilt + - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache + + # Cleanup + - echo "Doing cleanup for prebuilt\n\n" + - echo "Deleting specified files..." + - find ${BUILD_DIR} \( -name '*.a' -o -name '*.o' -o -name '*.os' -o -name '*.pyc' -o -name 'moc_*' -o -name '*.cc' -o -name "Jenkinsfile" -o -name "supercombo.onnx" -o -name ".sconsign.dblite" \) -print -delete + + - echo "Deleting Headers from selfdrive/ui path..." + - find ${BUILD_DIR}/selfdrive/ui -type f -name '*.h' -print -delete + + - echo "Deleting the following folders and subdirectories..." + - find "${BUILD_DIR}" -type d \( -path "*panda/board" -o -path "*panda/certs" -o -path "*panda/crypto" -o -path "*release" -o -path "*.github" -o -path "*selfdrive/ui/replay" -o -path "*__pycache__" \) -print -exec rm -rf {} + + + # Move back signed panda fw + - mkdir -p ${BUILD_DIR}/panda/board/obj + - cp ${CI_PROJECT_DIR}/panda/board/obj/panda.bin.signed ${BUILD_DIR}/panda/board/obj/panda.bin.signed + - cp ${CI_PROJECT_DIR}/panda/board/obj/panda_h7.bin.signed ${BUILD_DIR}/panda/board/obj/panda_h7.bin.signed + - cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda.bin + - cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda_h7.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda_h7.bin + - touch ${BUILD_DIR}/prebuilt + - mkdir -p ${OUTPUT_DIR} + - shopt -s dotglob && mv ${BUILD_DIR}/* ${OUTPUT_DIR} + artifacts: + paths: + - ${OUTPUT_DIR}/ + tags: [ 'sunnypilot', 'tici' ] + rules: + - if: $CI_MERGE_REQUEST_IID + when: manual + - if: $NEW_BRANCH + when: always + + +commit changes for public branch: + variables: + GIT_SUBMODULE_STRATEGY: normal + stage: commit_changes + dependencies: + - build + before_script: + - 'apk update && apk upgrade' + - 'apk add git bash openssh' + - 'eval $(ssh-agent -s)' + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - 'mkdir -p ~/.ssh/' + - 'chmod 700 ~/.ssh' + - 'ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts' # Adding gitlab to trusted + - 'chmod 644 ~/.ssh/known_hosts' + - *default_before_script + script: + - GIT_ORIGIN=$(echo "${CI_REPOSITORY_URL}" | sed -e 's|https\?://gitlab-ci-token:.*@|ssh://git@|g') + - echo "${GIT_ORIGIN}" + - echo "Calling to publish [${CI_DIR}/publish.sh ${CI_PROJECT_DIR} ${OUTPUT_DIR} ${NEW_BRANCH} ${VERSION} ${GIT_ORIGIN}]" + - git config --global --add safe.directory ${OUTPUT_DIR} + - $CI_DIR/publish.sh "${CI_PROJECT_DIR}" "${OUTPUT_DIR}" "${NEW_BRANCH}" "${VERSION}" "${GIT_ORIGIN}" + rules: + - if: $CI_MERGE_REQUEST_IID && $NEW_BRANCH + when: on_success + - if: $CI_MERGE_REQUEST_IID == "null" && $NEW_BRANCH + when: on_success + - when: never + +publish to github: + variables: + GIT_SUBMODULE_STRATEGY: normal + stage: publish + dependencies: + - build + before_script: + - 'apk update && apk upgrade' + - 'apk add git bash openssh' + - 'eval $(ssh-agent -s)' + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - + - 'mkdir -p ~/.ssh/' + - 'chmod 700 ~/.ssh' + - 'ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts' # Adding gitlab to trusted + - 'chmod 644 ~/.ssh/known_hosts' + - *default_before_script + script: + - echo "This part is not yet implemented intentionally, just an example" + rules: + - if: $NEW_BRANCH + when: manual diff --git a/SConstruct b/SConstruct index a086f8448a..2889460cd7 100644 --- a/SConstruct +++ b/SConstruct @@ -230,7 +230,8 @@ if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') # Setup cache dir -cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache' +cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir) CacheDir(cache_dir) Clean(["."], cache_dir) diff --git a/release/ci/build.sh b/release/ci/build.sh new file mode 100755 index 0000000000..3502f31c61 --- /dev/null +++ b/release/ci/build.sh @@ -0,0 +1,45 @@ +# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON + +#!/usr/bin/env bash + +# set -e + +# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# cd $DIR + +# # Take parameters as arguments +# SOURCE_DIR=$1 +# OUTPUT_DIR=$2 + +# # Check parameters +# if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then +# echo "Error: No source or output directory provided." +# exit 1 +# fi + +# if [ -z "$TARGETPLATFORM" ]; then +# # Detect the platform (e.g., x86_64, aarch64, etc.) +# platform=$(uname -m) + +# # Map to Docker's platform syntax +# if [ "$platform" = "x86_64" ]; then +# TARGETPLATFORM="linux/amd64" +# elif [ "$platform" = "aarch64" ]; then +# TARGETPLATFORM="linux/arm64" +# # Add here more elif statements for other architectures if needed. +# else +# echo "Unknown platform: $platform" +# exit 1 +# fi + +# export TARGETPLATFORM +# fi + +# echo "TARGETPLATFORM IS: [${TARGETPLATFORM}]" + +# # Let's bring the submodules! +# git submodule update --init --recursive + +# # Build +# docker compose -f ${SOURCE_DIR}/docker-compose.yml build +# docker compose -f ${SOURCE_DIR}/docker-compose.yml up diff --git a/release/ci/build_and_publish.sh b/release/ci/build_and_publish.sh new file mode 100755 index 0000000000..4063a54bd7 --- /dev/null +++ b/release/ci/build_and_publish.sh @@ -0,0 +1,27 @@ +# UNUSED YET BUT I WANT TO KEEP IT BECAUSE I INTEND TO USE IT SOON + +# #!/usr/bin/env bash +# DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +# cd $DIR + +# DEV_BRANCH=$1 +# GIT_ORIGIN=$2 +# if [ -z "$DEV_BRANCH" ]; then +# DEV_BRANCH="dev-c3-test" +# fi + +# if [ -z "$GIT_ORIGIN" ]; then +# GIT_ORIGIN="git@github.com:devtekve/openpilot-og.git" #This is a default that should be changed +# echo "No GIT_ORIGIN provided, we will use the default [${GIT_ORIGIN}]" +# fi + +# VERSION=$(date '+%Y.%m.%d') +# SOURCE_DIR="$(git rev-parse --show-toplevel)" +# OUTPUT_DIR=${SOURCE_DIR}/output + + +# echo "Calling to build [${DIR}/build.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${VERSION}]" +# $DIR/build.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${VERSION}" + +# echo "Calling to publish [${DIR}/publish.sh ${SOURCE_DIR} ${OUTPUT_DIR} ${DEV_BRANCH} ${VERSION} ${GIT_ORIGIN}]" +# $DIR/publish.sh "${SOURCE_DIR}" "${OUTPUT_DIR}" "${DEV_BRANCH}" "${VERSION}" "${GIT_ORIGIN}" diff --git a/release/ci/install_gitlab_runner.sh b/release/ci/install_gitlab_runner.sh new file mode 100755 index 0000000000..c8bac10c39 --- /dev/null +++ b/release/ci/install_gitlab_runner.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# We need RW for the install process +sudo mount -o remount rw / + +# Ensure filesystem is remounted as read-only on script exit +trap "sudo mount -o remount ro /" EXIT + +# Define directories +BASE_DIR="/data/gitlab" +BIN_DIR="$BASE_DIR/bin" +CONFIG_DIR="$BASE_DIR" +BUILDS_DIR="$BASE_DIR/builds" +OPENPILOT_DIR="$BASE_DIR/openpilot" +LOGS_DIR="$BASE_DIR/logs" +CACHE_DIR="$BASE_DIR/cache" +GITLAB_RUNNER_USERNAME="gitlab-runner" +GROUPS_NEEDED="comma,gpu,gpio" + +# Create necessary directories +sudo mkdir -p "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" + +# Download the GitLab Runner binary +sudo curl -L --output "$BIN_DIR/gitlab-runner" "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64" + +# Give it permission to execute +sudo chmod +x "$BIN_DIR/gitlab-runner" + +# Create a GitLab Runner user +sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${BASE_DIR} ${GITLAB_RUNNER_USERNAME} --shell /bin/bash -G ${GROUPS_NEEDED} || sudo usermod -aG ${GROUPS_NEEDED} gitlab-runner + +# Clean bash_logout as it break gitlab pipelines +sudo truncate -s 0 ${BASE_DIR}/.bash_logout + +# Create a configuration file +cat </dev/null && pwd)" +cd $DIR + +# Take parameters as arguments +SOURCE_DIR=$1 +OUTPUT_DIR=$2 +DEV_BRANCH=$3 +VERSION=$4 +GIT_ORIGIN=$5 + +# Check parameters +if [ -z "$SOURCE_DIR" ] || [ -z "$OUTPUT_DIR" ]; then + echo "Error: No source or output directory provided." + exit 1 +fi + +if [ -z "$DEV_BRANCH" ] || [ -z "$VERSION" ]; then + echo "Error: No dev branch or version provided." + exit 1 +fi + +if [ -z "$GIT_ORIGIN" ]; then + echo "Error: No GIT_ORIGIN provided" + exit 1 +fi + +# "Tagging" +echo "#define COMMA_VERSION \"$VERSION-dev\"" > ${OUTPUT_DIR}/common/version.h + +## set git identity +#source $DIR/identity.sh +#export GIT_SSH_COMMAND="ssh -i /data/gitkey" + +echo "[-] Setting up repo T=$SECONDS" +cd $OUTPUT_DIR +git init + +# set git username/password +#source /data/identity.sh + +git rm -rf $OUTPUT_DIR/.git || true # Doing cleanup, but it might fail if the .git doesn't exist or not allowed to delete +git remote remove origin || true # ensure cleanup +git remote add origin $GIT_ORIGIN +#git push origin -d $DEV_BRANCH || true # Ensuring we delete the remote branch if it exists as we are wiping it out +git fetch origin $DEV_BRANCH || (git checkout -b $DEV_BRANCH && git commit --allow-empty -m "sunnypilot v$VERSION release" && git push -u origin $DEV_BRANCH) + +echo "[-] committing version $VERSION T=$SECONDS" +git add -f . +git commit -a -m "sunnypilot v$VERSION release" +git branch --set-upstream-to=origin/$DEV_BRANCH + +# include source commit hash and build date in commit +GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) +DATETIME=$(date '+%Y-%m-%dT%H:%M:%S') +SP_VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}') + +# Add built files to git +git add -f . +git commit --amend -m "sunnypilot v$VERSION +version: sunnypilot v$SP_VERSION release +date: $DATETIME +master commit: $GIT_HASH +" +git branch -m $DEV_BRANCH + +# Push! +echo "[-] pushing T=$SECONDS" +git push -f origin $DEV_BRANCH diff --git a/selfdrive/manager/gitlab_runner.sh b/selfdrive/manager/gitlab_runner.sh new file mode 100644 index 0000000000..08774d8992 --- /dev/null +++ b/selfdrive/manager/gitlab_runner.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Define the service name +SERVICE_NAME="gitlab-runner" + +# Function to control the service +control_service() { + local action=$1 # Store the function argument in a local variable + sudo systemctl $action ${SERVICE_NAME} +} + +# Check for required argument +if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then + echo "Usage: $0 {start|stop}" + exit 1 +fi + +# Store the script argument in a descriptive variable +ACTION=$1 + +# Trap EXIT signal (Ctrl+C) and stop the service +trap 'control_service stop' SIGINT SIGKILL EXIT + +# Enter the main loop +while true; do + control_service $ACTION # Call the function with the specified action + sleep 1 # Pause before the next iteration +done diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index fd1ba29868..cdcefd5169 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -95,4 +95,11 @@ procs = [ PythonProcess("webjoystick", "tools.bodyteleop.web", notcar), ] +if os.path.exists("./gitlab_runner.sh") and True: # Of course and True is always true. Placeholder for a param :D + # Only devs! + procs += [ + NativeProcess("gitlab_runner_start", "selfdrive/manager", ["./gitlab_runner.sh", "start"], only_offroad, sigkill=False), + NativeProcess("gitlab_runner_stop", "selfdrive/manager", ["./gitlab_runner.sh", "stop"], only_onroad, sigkill=False) + ] + managed_processes = {p.name: p for p in procs}