From bcf6cd8e1a8a7c37a81b25931f9cb5094dcbe8ef Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 11 Nov 2023 08:31:51 +0100 Subject: [PATCH 1/5] Add sudo permissions to gitlab-runner This update adds sudo permissions to the gitlab-runner user group. The .gitlab-ci.yml file has been altered to change user ownership of ${OUTPUT_DIR} in the after_script phase. In install_gitlab_runner.sh, we also add gitlab-runner to the sudo group and explicitly grant it NOPASSWD: ALL rights in the sudoers file. --- .gitlab-ci.yml | 2 ++ release/ci/install_gitlab_runner.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a93556b1ea..4a6b6436fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,6 +100,8 @@ build: - touch ${BUILD_DIR}/prebuilt - mkdir -p ${OUTPUT_DIR} - shopt -s dotglob && mv ${BUILD_DIR}/* ${OUTPUT_DIR} + after_script: + - sudo chown -R comma:comma ${OUTPUT_DIR} artifacts: paths: - ${OUTPUT_DIR}/ diff --git a/release/ci/install_gitlab_runner.sh b/release/ci/install_gitlab_runner.sh index c8bac10c39..840537fe61 100755 --- a/release/ci/install_gitlab_runner.sh +++ b/release/ci/install_gitlab_runner.sh @@ -15,7 +15,7 @@ OPENPILOT_DIR="$BASE_DIR/openpilot" LOGS_DIR="$BASE_DIR/logs" CACHE_DIR="$BASE_DIR/cache" GITLAB_RUNNER_USERNAME="gitlab-runner" -GROUPS_NEEDED="comma,gpu,gpio" +GROUPS_NEEDED="comma,gpu,gpio,sudo" # Create necessary directories sudo mkdir -p "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" @@ -28,6 +28,7 @@ sudo chmod +x "$BIN_DIR/gitlab-runner" # Create a GitLab Runner user sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${BASE_DIR} ${GITLAB_RUNNER_USERNAME} --shell /bin/bash -G ${GROUPS_NEEDED} || sudo usermod -aG ${GROUPS_NEEDED} gitlab-runner +grep -qxF 'gitlab-runner ALL=(ALL) NOPASSWD: ALL' /etc/sudoers || echo 'gitlab-runner ALL=(ALL) NOPASSWD: ALL' | sudo tee -a /etc/sudoers #Giving us SUDO rights # Clean bash_logout as it break gitlab pipelines sudo truncate -s 0 ${BASE_DIR}/.bash_logout From aad8cd295b19f5a46f9284561317dbc3dc1f78c5 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 11 Nov 2023 08:31:51 +0100 Subject: [PATCH 2/5] Update GitLab runner scripts and add sudo permissions The GitLab Runner installation and uninstallation scripts have been refined, including directory management improvements and user handling enhancements. The install script now exports the base directory for subprocess availability and validates script arguments presence. An uninstall script has been added for smoother user experience. Additionally, sudo permissions have been granted to the gitlab-runner user group in both scripts and the .gitlab-ci.yml file, enhancing workflow efficiency. --- .gitlab-ci.yml | 2 + release/ci/install_gitlab_runner.sh | 122 ++++++++++++++++---------- release/ci/uninstall_gitlab_runner.sh | 58 ++++++++++++ 3 files changed, 135 insertions(+), 47 deletions(-) create mode 100644 release/ci/uninstall_gitlab_runner.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a93556b1ea..4a6b6436fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -100,6 +100,8 @@ build: - touch ${BUILD_DIR}/prebuilt - mkdir -p ${OUTPUT_DIR} - shopt -s dotglob && mv ${BUILD_DIR}/* ${OUTPUT_DIR} + after_script: + - sudo chown -R comma:comma ${OUTPUT_DIR} artifacts: paths: - ${OUTPUT_DIR}/ diff --git a/release/ci/install_gitlab_runner.sh b/release/ci/install_gitlab_runner.sh index c8bac10c39..78e92490f4 100755 --- a/release/ci/install_gitlab_runner.sh +++ b/release/ci/install_gitlab_runner.sh @@ -1,80 +1,108 @@ #!/bin/bash +set -e -# We need RW for the install process -sudo mount -o remount rw / +# Check if script arguments are present, if not exit the script +if [ $# -eq 0 ]; then + echo "No arguments provided. A GitLab token is required to run this script." + exit 1 +fi -# Ensure filesystem is remounted as read-only on script exit -trap "sudo mount -o remount ro /" EXIT +# Constants +GITLAB_RUNNER_DOWNLOAD_URL="https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64" +GITLAB_RUNNER_USER_NAME="gitlab-runner" +USER_GROUPS="comma,gpu,gpio,sudo" +GITLAB_BASE_DIR="/data/gitlab" +GITLAB_BIN_DIR="${GITLAB_BASE_DIR}/bin" +GITLAB_BUILDS_DIR="${GITLAB_BASE_DIR}/builds" +GITLAB_LOGS_DIR="${GITLAB_BASE_DIR}/logs" +GITLAB_CACHE_DIR="${GITLAB_BASE_DIR}/cache" +GITLAB_OPENPILOT_DIR="${GITLAB_BASE_DIR}/openpilot" +SERVICE_NAME="gitlab-runner" -# Define directories -BASE_DIR="/data/gitlab" -BIN_DIR="$BASE_DIR/bin" -CONFIG_DIR="$BASE_DIR" -BUILDS_DIR="$BASE_DIR/builds" -OPENPILOT_DIR="$BASE_DIR/openpilot" -LOGS_DIR="$BASE_DIR/logs" -CACHE_DIR="$BASE_DIR/cache" -GITLAB_RUNNER_USERNAME="gitlab-runner" -GROUPS_NEEDED="comma,gpu,gpio" +create_gitlab_runner_directories() { + sudo mkdir -p "$GITLAB_BIN_DIR" "$GITLAB_BUILDS_DIR" "$GITLAB_LOGS_DIR" "$GITLAB_CACHE_DIR" "$GITLAB_OPENPILOT_DIR" + if [[ ! -d "/data/openpilot" ]]; then + sudo mkdir -p "/data/openpilot" + sudo chown -R comma:comma "/data/openpilot" + fi +} -# Create necessary directories -sudo mkdir -p "$BIN_DIR" "$BUILDS_DIR" "$LOGS_DIR" "$CACHE_DIR" "$OPENPILOT_DIR" +download_and_setup_gitlab_runner() { + sudo curl -L --output "$GITLAB_BIN_DIR/gitlab-runner" "$GITLAB_RUNNER_DOWNLOAD_URL" + sudo chmod +x "$GITLAB_BIN_DIR/gitlab-runner" +} -# Download the GitLab Runner binary -sudo curl -L --output "$BIN_DIR/gitlab-runner" "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64" +setup_gitlab_runner_user() { + sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${GITLAB_BASE_DIR} ${GITLAB_RUNNER_USER_NAME} --shell /bin/bash -G ${USER_GROUPS} || sudo usermod -aG ${USER_GROUPS} ${GITLAB_RUNNER_USER_NAME} + export GITLAB_BASE_DIR # Export it to make it available to sub-processes + sudo -u ${GITLAB_RUNNER_USER_NAME} bash -c "truncate -s 0 '${GITLAB_BASE_DIR}/.bash_logout'" +} -# Give it permission to execute -sudo chmod +x "$BIN_DIR/gitlab-runner" +create_sudoers_entry() { + sudo grep -qxF "${GITLAB_RUNNER_USER_NAME} ALL=(ALL) NOPASSWD: ALL" /etc/sudoers || echo "${GITLAB_RUNNER_USER_NAME} ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers +} -# Create a GitLab Runner user -sudo useradd --comment 'GitLab Runner' --create-home --home-dir ${BASE_DIR} ${GITLAB_RUNNER_USERNAME} --shell /bin/bash -G ${GROUPS_NEEDED} || sudo usermod -aG ${GROUPS_NEEDED} gitlab-runner - -# Clean bash_logout as it break gitlab pipelines -sudo truncate -s 0 ${BASE_DIR}/.bash_logout - -# Create a configuration file -cat < Date: Sat, 11 Nov 2023 12:38:16 +0100 Subject: [PATCH 3/5] Add new Build Release and Build Debug configurations Two new run configurations for Build Release and Build Debug have been added. These configurations include specific options, like location of working directory and external build option. Each configuration includes a unique name and target name. --- .run/Build Debug.run.xml | 10 ++++++++++ .run/Build Release.run.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .run/Build Debug.run.xml create mode 100644 .run/Build Release.run.xml diff --git a/.run/Build Debug.run.xml b/.run/Build Debug.run.xml new file mode 100644 index 0000000000..6a80d76b97 --- /dev/null +++ b/.run/Build Debug.run.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/.run/Build Release.run.xml b/.run/Build Release.run.xml new file mode 100644 index 0000000000..8231638ef7 --- /dev/null +++ b/.run/Build Release.run.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file From c073305bae8f60caf806ce3ce1c612c838df60aa Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 11 Nov 2023 12:55:15 +0100 Subject: [PATCH 4/5] Update build configurations for Poetry SCons The build configurations for both Debug and Release modes have been updated to use the Poetry SCons tool. A new customTargets.xml file has been added to specify the build actions for Poetry SCons in Debug and Release targets. --- .idea/customTargets.xml | 25 +++++++++++++++++++++++++ .run/Build Debug.run.xml | 2 +- .run/Build Release.run.xml | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 .idea/customTargets.xml diff --git a/.idea/customTargets.xml b/.idea/customTargets.xml new file mode 100644 index 0000000000..457f9942d0 --- /dev/null +++ b/.idea/customTargets.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.run/Build Debug.run.xml b/.run/Build Debug.run.xml index 6a80d76b97..a7b47c9011 100644 --- a/.run/Build Debug.run.xml +++ b/.run/Build Debug.run.xml @@ -1,5 +1,5 @@ - + diff --git a/.run/Build Release.run.xml b/.run/Build Release.run.xml index 8231638ef7..cbf6343e87 100644 --- a/.run/Build Release.run.xml +++ b/.run/Build Release.run.xml @@ -1,5 +1,5 @@ - + From 29c65ff033611da0d017b4c9ca0cad18d3d0b530 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 11 Nov 2023 13:45:44 +0100 Subject: [PATCH 5/5] Dont create folder with sudo --- release/ci/install_gitlab_runner.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/ci/install_gitlab_runner.sh b/release/ci/install_gitlab_runner.sh index c89b32ffe9..a44c13cc92 100755 --- a/release/ci/install_gitlab_runner.sh +++ b/release/ci/install_gitlab_runner.sh @@ -21,7 +21,7 @@ SERVICE_NAME="gitlab-runner" create_gitlab_runner_directories() { sudo mkdir -p "$GITLAB_BIN_DIR" "$GITLAB_BUILDS_DIR" "$GITLAB_LOGS_DIR" "$GITLAB_CACHE_DIR" "$GITLAB_OPENPILOT_DIR" - sudo mkdir -p "/data/openpilot" + mkdir -p "/data/openpilot" sudo chown -R comma:comma "/data/openpilot" }