From 4268d7a19c82c2731b019ede045626f675d81cb3 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 29 Mar 2025 22:34:31 +0100 Subject: [PATCH 1/2] Events: Refactor `OnroadEventSP` structure and add upstream cereal validation (#722) * Refactor OnroadEventSP structure to contain list of events A restructuring of the OnroadEventSP structure has been undertaken to accommodate a list of 'Event' substructures. The change is reflected in different files where OnroadEventSP is used. This update allows for more efficient management of multiple events by grouping them together under the revised OnroadEventSP structure. * Rename `OnroadEventSP` to `OnroadEventsSP` across codebase. Updated all references to `OnroadEventSP` to ensure consistency with the renamed struct `OnroadEventsSP`. This change improves code clarity and aligns naming conventions across modules. * Add optional debug logging to schema validation script Introduced a `DEBUG` flag and a `print_debug` function to streamline debug output management. This replaces direct `print` calls with conditional logging to control verbosity during execution. Refactor structural validation logic in cereal test Simplify the iteration over read_instances to streamline structural validation. Removed redundant comparisons and improved error handling to detect unreadable fields more effectively. Updated error messages for better clarity during debugging. Update build command to include 'cereal' target in CI Modified the scons build command in selfdrive_tests workflow to explicitly build the 'cereal' target. This ensures necessary components are included during the CI process, improving reliability and consistency. Added workflow for cereal validation artifacts generation and validation against upstream This commit encompasses significant changes to .github/workflows/selfdrive_tests.yaml, including the addition of two new jobs. One is responsible for 'Generating cereal validation artifacts' and the other for 'Validating cereal with Upstream'. This includes generating cereal schemas, building openpilot, and running validation schema instances against master. Furthermore, a new Python script (validate_sp_cereal_upstream.py) was also added to perform cereal schema instance generation and validation. These changes aim to enhance the testing process, ensuring schema compatibility and integration quality. * Relocate cereal validation to a dedicated GitHub workflow This commit introduces a distinct GitHub workflow for cereal validation named 'cereal_validation.yaml'. This workflow includes two jobs: one for generating cereal validation artifacts and another for validating cereal with the upstream project. Previously, these operations were included as separate jobs in 'selfdrive_tests.yaml'. However, the decoupling in this commit allows for a better organization of GitHub workflows within the project. Additionally, this separation allows these workflows to be individually configured and run, providing a greater degree of flexibility in managing our continuous integration activities. * Rename workflow to "cereal validation" for clarity. Updated the workflow name in the GitHub Actions configuration to better reflect its purpose. This change improves maintainability and ensures clearer identification of the workflow's function. * Add LFS configuration and GitLab SSH setup to workflow Integrate GitLab LFS handling by configuring LFS URLs and enabling SSH setup. This includes adding public GitLab keys and updating the workflow to support secure connections for LFS operations. Ensures proper handling of large files and seamless integration with GitLab. * rename * format --------- Co-authored-by: Jason Wen --- .github/workflows/cereal_validation.yaml | 77 ++++++ cereal/custom.capnp | 28 ++- cereal/log.capnp | 2 +- .../tests/validate_sp_cereal_upstream.py | 222 ++++++++++++++++++ selfdrive/selfdrived/selfdrived.py | 4 +- sunnypilot/selfdrive/selfdrived/events.py | 2 +- 6 files changed, 319 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/cereal_validation.yaml create mode 100755 cereal/messaging/tests/validate_sp_cereal_upstream.py diff --git a/.github/workflows/cereal_validation.yaml b/.github/workflows/cereal_validation.yaml new file mode 100644 index 0000000000..5d75b6bc13 --- /dev/null +++ b/.github/workflows/cereal_validation.yaml @@ -0,0 +1,77 @@ +name: cereal validation + +on: + push: + branches: + - master + - master-new + pull_request: + paths: + - 'cereal/**' + workflow_dispatch: + workflow_call: + inputs: + run_number: + default: '1' + required: true + type: string + +concurrency: + group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }} + cancel-in-progress: true + +env: + PYTHONWARNINGS: error + BASE_IMAGE: openpilot-base + BUILD: selfdrive/test/docker_build.sh base + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + +jobs: + generate_cereal_artifact: + name: Generate cereal validation artifacts + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot + run: ${{ env.RUN }} "scons -j$(nproc) cereal" + - name: Generate the log file + run: | + ${{ env.RUN }} "cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin" && \ + ls -la + ls -la cereal/messaging/tests + - name: 'Prepare artifact' + run: | + mkdir -p "cereal/messaging/tests/cereal_validations" + cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py" + cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin" + - name: 'Upload Artifact' + uses: actions/upload-artifact@v4 + with: + name: cereal_validations + path: cereal/messaging/tests/cereal_validations + + validate_cereal_with_upstream: + name: Validate cereal with Upstream + runs-on: ubuntu-24.04 + needs: generate_cereal_artifact + steps: + - uses: actions/checkout@v4 + with: + repository: 'commaai/openpilot' + submodules: true + ref: "refs/heads/master" + - uses: ./.github/workflows/setup-with-retry + - name: Build openpilot + run: ${{ env.RUN }} "scons -j$(nproc) cereal" + - name: Download build artifacts + uses: actions/download-artifact@v4 + with: + name: cereal_validations + path: cereal/messaging/tests/cereal_validations + - name: 'Run the validation' + run: | + chmod +x cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py + ${{ env.RUN }} "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f cereal/messaging/tests/cereal_validations/schema_instances.bin" diff --git a/cereal/custom.capnp b/cereal/custom.capnp index 9c280349c1..8d09392fc4 100644 --- a/cereal/custom.capnp +++ b/cereal/custom.capnp @@ -102,19 +102,23 @@ struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 { } struct OnroadEventSP @0xda96579883444c35 { - name @0 :EventName; + events @0 :List(Event); - # event types - enable @1 :Bool; - noEntry @2 :Bool; - warning @3 :Bool; # alerts presented only when enabled or soft disabling - userDisable @4 :Bool; - softDisable @5 :Bool; - immediateDisable @6 :Bool; - preEnable @7 :Bool; - permanent @8 :Bool; # alerts presented regardless of openpilot state - overrideLateral @10 :Bool; - overrideLongitudinal @9 :Bool; + struct Event { + name @0 :EventName; + + # event types + enable @1 :Bool; + noEntry @2 :Bool; + warning @3 :Bool; # alerts presented only when enabled or soft disabling + userDisable @4 :Bool; + softDisable @5 :Bool; + immediateDisable @6 :Bool; + preEnable @7 :Bool; + permanent @8 :Bool; # alerts presented regardless of openpilot state + overrideLateral @10 :Bool; + overrideLongitudinal @9 :Bool; + } enum EventName { lkasEnable @0; diff --git a/cereal/log.capnp b/cereal/log.capnp index 0d1fcf19a3..8595bcda02 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -2579,7 +2579,7 @@ struct Event { selfdriveStateSP @107 :Custom.SelfdriveStateSP; modelManagerSP @108 :Custom.ModelManagerSP; longitudinalPlanSP @109 :Custom.LongitudinalPlanSP; - onroadEventsSP @110 :List(Custom.OnroadEventSP); + onroadEventsSP @110 :Custom.OnroadEventSP; carParamsSP @111 :Custom.CarParamsSP; carControlSP @112 :Custom.CarControlSP; backupManagerSP @113 :Custom.BackupManagerSP; diff --git a/cereal/messaging/tests/validate_sp_cereal_upstream.py b/cereal/messaging/tests/validate_sp_cereal_upstream.py new file mode 100755 index 0000000000..9ccd6533ce --- /dev/null +++ b/cereal/messaging/tests/validate_sp_cereal_upstream.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +import argparse +import sys +from typing import Any, List, Tuple + +DEBUG = False + + +def print_debug(string: str) -> None: + if DEBUG: + print(string) + + +def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any: + """ + Create a new instance of a schema type, handling different field types. + + Args: + struct: The Cap'n Proto schema structure + prop: A tuple containing the field name and field metadata + + Returns: + A new initialized schema instance + """ + struct_instance = struct.new_message() + field_name, field_metadata = prop + + try: + field_type = field_metadata.proto.slot.type.which() + + # Initialize different types of fields + if field_type in ('list', 'text', 'data'): + struct_instance.init(field_name, 1) + print_debug(f"Initialized list/text/data field: {field_name}") + elif field_type in ('struct', 'object'): + struct_instance.init(field_name) + print_debug(f"Initialized struct/object field: {field_name}") + + return struct_instance + + except Exception as e: + print(f"Error creating instance for {field_name}: {e}") + return None + + +def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]: + """ + Retrieve all fields from a given schema structure. + + Args: + schema_struct: The Cap'n Proto schema structure + + Returns: + A list of field names and their metadata + """ + try: + # Get all fields from the schema + schema_fields = list(schema_struct.schema.fields.items()) + + print_debug("Discovered schema fields:") + for field_name, field_metadata in schema_fields: + print_debug(f"- {field_name}") + + return schema_fields + + except Exception as e: + print(f"Error retrieving schema fields: {e}") + return [] + + +def generate_schema_instances(schema_struct: Any) -> List[Any]: + """ + Generate instances for all fields in a given schema. + + Args: + schema_struct: The Cap'n Proto schema structure + + Returns: + A list of schema instances + """ + schema_fields = get_schema_fields(schema_struct) + instances = [] + + for field_prop in schema_fields: + try: + instance = create_schema_instance(schema_struct, field_prop) + if instance is not None: + instances.append(instance) + except Exception as e: + print(f"Skipping field due to error: {e}") + + print(f"Generated {len(instances)} schema instances") + return instances + + +def persist_instances(instances: List[Any], filename: str) -> None: + """ + Write schema instances to a binary file. + + Args: + instances: List of schema instances + filename: Output file path + """ + try: + with open(filename, 'wb') as f: + for instance in instances: + f.write(instance.to_bytes()) + + print(f"Successfully wrote {len(instances)} instances to {filename}") + + except Exception as e: + print(f"Error persisting instances: {e}") + sys.exit(1) + + +def read_instances(filename: str, schema_type: Any) -> List[Any]: + """ + Read schema instances from a binary file. + + Args: + filename: Input file path + schema_type: The schema type to use for reading + + Returns: + A list of read schema instances + """ + try: + with open(filename, 'rb') as f: + data = f.read() + + instances = list(schema_type.read_multiple_bytes(data)) + + print(f"Read {len(instances)} instances from {filename}") + return instances + + except Exception as e: + print(f"Error reading instances: {e}") + sys.exit(1) + + +def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool: + """ + Compare original and read-back instances to detect potential breaking changes. + + Args: + original_instances: List of originally generated instances + read_instances: List of instances read back from file + + Returns: + Boolean indicating whether schemas appear compatible + """ + if len(original_instances) != len(read_instances): + print("❌ Schema Compatibility Warning: Instance count mismatch") + return False + + compatible = True + for struct in read_instances: + try: + getattr(struct, struct.which()) # Attempting to access the field to validate readability + except Exception as e: + print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}") + compatible = False + + return compatible + + +def main(): + """ + CLI entry point for schema compatibility testing. + """ + # Setup argument parser + parser = argparse.ArgumentParser( + description='Cap\'n Proto Schema Compatibility Testing Tool', + epilog='Test schema compatibility by generating and reading back instances.' + ) + + # Add mutually exclusive group for generation or reading mode + mode_group = parser.add_mutually_exclusive_group(required=True) + mode_group.add_argument('-g', '--generate', action='store_true', + help='Generate schema instances') + mode_group.add_argument('-r', '--read', action='store_true', + help='Read and validate schema instances') + + # Common arguments + parser.add_argument('-f', '--file', + default='schema_instances.bin', + help='Output/input binary file (default: schema_instances.bin)') + + # Parse arguments + args = parser.parse_args() + + # Import the schema dynamically + try: + from cereal import log + schema_type = log.Event + except ImportError: + print("Error: Unable to import schema. Ensure 'cereal' is installed.") + sys.exit(1) + + # Execute based on mode + if args.generate: + print("🔧 Generating Schema Instances") + instances = generate_schema_instances(schema_type) + persist_instances(instances, args.file) + print("✅ Instance generation complete") + + elif args.read: + print("🔍 Reading and Validating Schema Instances") + generated_instances = generate_schema_instances(schema_type) + read_back_instances = read_instances(args.file, schema_type) + + # Compare schemas + if compare_schemas(generated_instances, read_back_instances): + print("✅ Schema Compatibility: No breaking changes detected") + sys.exit(0) + else: + print("❌ Potential Schema Breaking Changes Detected") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index 9a28a36169..b6a3348d5a 100755 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -509,9 +509,9 @@ class SelfdriveD(CruiseHelper): # onroadEventsSP - logged every second or on change if (self.sm.frame % int(1. / DT_CTRL) == 0) or (self.events_sp.names != self.events_sp_prev): - ce_send_sp = messaging.new_message('onroadEventsSP', len(self.events_sp)) + ce_send_sp = messaging.new_message('onroadEventsSP') ce_send_sp.valid = True - ce_send_sp.onroadEventsSP = self.events_sp.to_msg() + ce_send_sp.onroadEventsSP.events = self.events_sp.to_msg() self.pm.send('onroadEventsSP', ce_send_sp) self.events_sp_prev = self.events_sp.names.copy() diff --git a/sunnypilot/selfdrive/selfdrived/events.py b/sunnypilot/selfdrive/selfdrived/events.py index ac564692a8..7767a6db70 100644 --- a/sunnypilot/selfdrive/selfdrived/events.py +++ b/sunnypilot/selfdrive/selfdrived/events.py @@ -26,7 +26,7 @@ class EventsSP(EventsBase): return EVENT_NAME_SP[event] def get_event_msg_type(self): - return custom.OnroadEventSP + return custom.OnroadEventSP.Event EVENTS_SP: dict[int, dict[str, Alert | AlertCallbackType]] = { From 6b3f75bbf0784fa516e5030de5d9f2cb68ca041e Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 29 Mar 2025 22:54:49 +0100 Subject: [PATCH 2/2] CI: refactor Squash and Merge with simplified branch merging (#726) * Refactor squash and merge script for improved simplicity Simplified the squash_and_merge.py script by replacing redundant utility functions and consolidating logic. Enhanced usability by aligning command-line arguments and leveraging streamlined git operations to improve maintainability and reliability. * Fix argument names in squash PR script Renamed CLI arguments from '--base' and '--source' to '--target' and '--base' to align with expected input format. This ensures the script runs correctly with proper argument mapping. * Fix incorrect base branch argument in squash script Updated the `--base` argument to use `source_branch` instead of `branch` to ensure the squash script processes the correct base branch. Also adjusted the command to include `branch` as a separate argument for clarity and correctness. * Reset to a clean state after squash error. Add a `git reset --hard` command to ensure the repository returns to a clean state after encountering errors during the squash and merge process. This prevents lingering changes from affecting subsequent operations. * Improve error handling in squash_and_merge_prs.py Capture and display both stdout and stderr in error cases to provide more informative feedback. Adjust the PR comment to include available output for better debugging. * Refactor PR squash process to enhance error handling. Modify subprocess handling to use `result.returncode` for error checks instead of relying on exceptions. Consolidate error output retrieval and logging for better clarity, while maintaining the workflow for resetting changes on failure. * Fix incorrect return in PR processing loop Replaced `return` with `continue` to ensure all PRs in the loop are processed before exiting. This prevents premature termination of the function and ensures accurate success count reporting. * Simplify subprocess output handling in squash_and_merge.py Replaced labeled print statements with direct output of stdout and stderr. This change ensures cleaner logs and remains consistent with the function's purpose of output handling during subprocess execution. * Update subprocess.run calls to use capture_output parameter Replaced `stdout` and `stderr` with the `capture_output` parameter for cleaner and more concise subprocess handling. Also removed extraneous whitespace for improved code readability. * testing moving the squash script given that it's called iteratively and switching branch might miss it * format --------- Co-authored-by: Jason Wen --- .../sunnypilot-master-dev-c3-prep.yaml | 12 +- release/ci/squash_and_merge.py | 381 ++---------------- release/ci/squash_and_merge_prs.py | 27 +- 3 files changed, 59 insertions(+), 361 deletions(-) diff --git a/.github/workflows/sunnypilot-master-dev-c3-prep.yaml b/.github/workflows/sunnypilot-master-dev-c3-prep.yaml index 54b80f3627..0a6a76099f 100644 --- a/.github/workflows/sunnypilot-master-dev-c3-prep.yaml +++ b/.github/workflows/sunnypilot-master-dev-c3-prep.yaml @@ -36,7 +36,7 @@ jobs: run: | git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - + - name: Set up SSH uses: webfactory/ssh-agent@v0.9.0 with: @@ -63,10 +63,10 @@ jobs: echo "Source branch ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }} does not exist!" exit 1 fi - + # Make sure we have the latest source branch git fetch origin ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }} - + # Check if target branch exists if ! git ls-remote --heads origin ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} | grep -q "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"; then echo "Target branch ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} does not exist, creating it from ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}" @@ -110,17 +110,19 @@ jobs: } } }' -F label="is:pr is:open label:${PR_LABEL} sort:created-asc") - + echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Process PRs run: | + cp ${{ github.workspace }}/release/ci/squash_and_merge.py /tmp/squash_and_merge.py && \ + chmod +x /tmp/squash_and_merge.py && \ python3 ${{ github.workspace }}/release/ci/squash_and_merge_prs.py \ --pr-data '${{ steps.get-prs.outputs.PR_LIST }}' \ --target-branch ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} \ - --squash-script-path '${{ github.workspace }}/release/ci/squash_and_merge.py' + --squash-script-path '/tmp/squash_and_merge.py' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/release/ci/squash_and_merge.py b/release/ci/squash_and_merge.py index 7ab088801b..f41e03903d 100755 --- a/release/ci/squash_and_merge.py +++ b/release/ci/squash_and_merge.py @@ -1,360 +1,53 @@ #!/usr/bin/env python3 -import argparse import subprocess import sys -import shutil -import signal -import contextlib -import tempfile -import os +import argparse -def run_command(command: str) -> tuple[int, str, str]: - """Run a shell command and return exit code, stdout, and stderr.""" - process = subprocess.Popen( - command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True - ) - stdout, stderr = process.communicate() - return process.returncode, stdout.strip(), stderr.strip() - - -def is_gh_available() -> bool: - """Check if GitHub CLI is available.""" - return shutil.which('gh') is not None - - -def get_current_branch() -> str | None: - """Get the name of the current git branch.""" - code, output, error = run_command("git rev-parse --abbrev-ref HEAD") - if code != 0: - print(f"Error getting current branch: {error}") - return None - return output - - -def backup_branch(branch_name: str) -> bool: - """Create a backup of the current branch.""" - backup_name = f"{branch_name}-backup-$(date +%Y%m%d_%H%M%S)" - code, _, error = run_command(f"git branch {backup_name}") - if code != 0: - print(f"Error creating backup branch: {error}") - return False - print(f"Created backup branch: {backup_name}") - return True - - -def get_commit_messages(source_branch: str, target_branch: str) -> list[str] | None: - """Get all commit messages between source and target branches.""" - code, output, error = run_command(f"git log {target_branch}..{source_branch} --format=%B") - if code != 0: - print(f"Error getting commit messages: {error}") - return None - return [msg.strip() for msg in output.splitlines() if msg and not msg.startswith('Merge')] - - -def get_pr_info(branch_name: str) -> str | None: - """Get PR title using GitHub CLI.""" - if not is_gh_available(): - print("Warning: GitHub CLI not found. Install it to auto-fetch PR titles:") - print(" https://cli.github.com/") - return None - - # Try to get PR info using gh cli - code, output, error = run_command(f"gh pr view --json title --jq .title {branch_name}") - if code != 0: - print(f"No open PR found for branch '{branch_name}'") - return None - - return output - - -def create_squash_message(pr_title: str | None, commit_messages: list[str], source_branch: str) -> str: - """Create a squash commit message from PR title and commit messages.""" - parts = [] - - # Add PR title if provided - if pr_title: - parts.append(pr_title) - else: - parts.append(f"Squashed changes from {source_branch}") - parts.append("") # Empty line after title - - # Add original commits section - if commit_messages: - parts.append("Original commits:") - parts.append("") # Empty line before list - parts.extend(f"* {msg}" for msg in commit_messages) - - return '\n'.join(parts) - - -def prompt_for_title() -> str: - """Prompt user for a commit title.""" - return input("Enter commit title (or press Enter to use default): ").strip() - - -@contextlib.contextmanager -def workspace_manager(original_branch: str): - """Context manager to handle workspace state and cleanup.""" - stash_created = False - stash_restored = False - temp_branch: str | None = None - - def cleanup_handler(signum=None, frame=None): - """Clean up workspace state.""" - nonlocal temp_branch, stash_created, stash_restored - try: - if signum and stash_restored: - # If we're handling Ctrl+C but stash was already restored, - # just clean up branches and exit - current = get_current_branch() - if current and current != original_branch: - run_command(f"git checkout {original_branch}") - if temp_branch: - run_command(f"git branch -D {temp_branch}") - print("\nOperation interrupted, but changes were already restored.") - sys.exit(3) - - # First, switch back to original branch - current = get_current_branch() - if current and current != original_branch: - run_command(f"git checkout {original_branch}") - - # Then clean up temp branch - if temp_branch: - run_command(f"git branch -D {temp_branch}") - - # Finally, restore stash if needed - AFTER switching branches - if stash_created and not stash_restored: - print("Restoring your uncommitted changes...") - code, stash_list, _ = run_command("git stash list") - if code == 0 and "Automatic stash by squash script" in stash_list: - run_command("git stash pop") - stash_restored = True - stash_created = False - - if signum: - print("\nOperation interrupted. Cleaned up and restored original state.") - sys.exit(4) - - except Exception as e: - print(f"Error during cleanup: {e}") - if signum: - sys.exit(5) - - try: - # Set up signal handlers - signal.signal(signal.SIGINT, cleanup_handler) - signal.signal(signal.SIGTERM, cleanup_handler) - - # Check for changes (including untracked files) - code, output, _ = run_command("git status --porcelain") - if output: - print("Stashing uncommitted changes...") - run_command("git stash push -u -m 'Automatic stash by squash script'") - stash_created = True - - yield lambda x: setattr(x, 'temp_branch', temp_branch) - - except Exception as e: - print(f"\nError occurred: {str(e)}") - cleanup_handler() - raise - finally: - cleanup_handler() - - -def create_commit_with_message(message: str) -> bool: - """Create a commit with the given message using a temporary file.""" - try: - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - f.write(message) - temp_path = f.name - - # Use the temporary file for the commit message - code, _, error = run_command(f"git commit -F {temp_path}") - os.unlink(temp_path) # Clean up the temp file - - if code != 0: - print(f"Error creating commit: {error}") - return False - return True - except Exception as e: - print(f"Error handling commit message: {e}") - if os.path.exists(temp_path): - os.unlink(temp_path) - return False - - -def squash_and_merge(source_branch: str, target_branch: str, manual_title: str | None, backup: bool = False, push: bool = False) -> bool: +def run_git_command(command, check=True): """ - Squash the source branch and merge into target branch. + Runs a git command and returns the trimmed stdout output. + Exits the script if the command fails. """ - # Get original branch right away - original_branch = get_current_branch() - if not original_branch: - return False - - class State: - temp_branch: str | None = None - - state = State() - - with workspace_manager(original_branch) as set_temp_branch: - # Validate source branch exists - code, _, error = run_command(f"git rev-parse --verify {source_branch}") - if code != 0: - print(f"Error: Source branch {source_branch} not found") - return False - - if source_branch == target_branch: - print(f"Error: Source and target branches cannot be the same ({source_branch})") - return False - - # Ensure target branch exists - code, _, error = run_command(f"git rev-parse --verify {target_branch}") - if code != 0: - print(f"Error: Target branch {target_branch} not found") - return False - - # Find merge base - code, merge_base, error = run_command(f"git merge-base {target_branch} {source_branch}") - if code != 0: - print(f"Error finding merge base: {error}") - return False - - # Create backup unless explicitly skipped - if backup and not backup_branch(source_branch): - return False - - # Get commit messages - commit_messages = get_commit_messages(source_branch, target_branch) - if commit_messages is None: - return False - - # Get title (priority: manual title > PR title > prompt user) - title = manual_title - if not title: - title = get_pr_info(source_branch) - if not title: - title = prompt_for_title() - - try: - # Create and switch to temporary branch - temp_branch = f"temp-squash-{source_branch}" - state.temp_branch = temp_branch - set_temp_branch(state) - - print(f"\nCreating temporary branch {temp_branch}...") - code, _, error = run_command(f"git checkout -b {temp_branch} {source_branch}") - if code != 0: - print(f"Error creating temp branch: {error}") - return False - - print("Preparing squash by resetting temporary branch to merge base...") - code, _, error = run_command(f"git reset --soft {merge_base}") - if code != 0: - print(f"Error resetting for squash: {error}") - return False - - # Create commit with message - print("Creating squash commit...") - squash_message = create_squash_message(title, commit_messages, source_branch) - if not create_commit_with_message(squash_message): - return False - - # Switch to target and try merge - print(f"\nSwitching to target branch {target_branch}...") - code, _, error = run_command(f"git checkout {target_branch}") - if code != 0: - print(f"Error checking out target branch: {error}") - return False - - print(f"Attempting to merge changes from {temp_branch}...") - code, _, error = run_command(f"git rebase {temp_branch}") - - if code != 0: - print(f"\nMerge failed with error: {error}") - print("\nThe squash was successful, and your changes are preserved in the temporary branch.") - print("To complete the merge manually, follow these steps:") - print(f"\n1. Your squashed changes are in branch: '{temp_branch}'") - print(f"2. The target branch is: '{target_branch}'") - print("\nTo resolve the conflicts:") - print(f" git checkout {target_branch}") - print(f" git merge {temp_branch}") - print(" # resolve conflicts in your editor") - print(" git add ") - print(" git commit") - print(f" git push origin {target_branch} # when ready to push") - print("\nTo clean up after successful merge:") - print(f" git branch -D {temp_branch}") - - # Make sure to abort the merge - print("\nAborting current merge attempt...") - run_command("git merge --abort") - - # Return to original branch, but keep temp branch - print(f"Returning to {original_branch}...") - run_command(f"git checkout {original_branch}") - return False - - # Clean up temp branch on success - run_command(f"git branch -D {temp_branch}") - - # Push if requested - if push: - code, _, error = run_command(f"git push origin {target_branch}") - if code != 0: - print(f"Error pushing to {target_branch}: {error}") - return False - print(f"Successfully pushed to {target_branch}") - else: - print(f"Changes squashed and merged into {target_branch} locally") - print(f"To push the changes: git push origin {target_branch}") - - # Return to original branch - code, _, error = run_command(f"git checkout {original_branch}") - if code != 0: - print(f"Warning: Failed to return to original branch: {error}") - return False - - return True - - except Exception as e: - print(f"Error during squash process: {e}") - return False + print(f"Running: {' '.join(command)}") + result = subprocess.run(command, capture_output=True, text=True) + if check and result.returncode != 0: + print(result.stdout.strip()) + print(result.stderr.strip()) + sys.exit(result.returncode) + return result.stdout.strip() def main(): - parser = argparse.ArgumentParser( - description='Squash branch and merge into target branch' - ) - parser.add_argument('--target', '-t', required=True, - help='Target branch to merge changes into') - parser.add_argument('--source', '-s', - help='Source branch to squash (default: current branch)') - parser.add_argument('--title', '-m', - help='Optional manual title (overrides PR title)') - parser.add_argument('--backup', action='store_true', - help='Creates a backup branch for the source branch') - parser.add_argument('--push', action='store_true', - help='Push changes to remote after squashing') + parser = argparse.ArgumentParser(description="Merge multiple branches with squash merges.") + parser.add_argument("--base", required=True, help="The base branch name from which the target branch will be created.") + parser.add_argument("--target", required=True, help="The target branch name to merge into.") + parser.add_argument("--title", required=False, help="Title for the commit") - args, unknown = parser.parse_known_args() + parser.add_argument("branches", nargs="+", help="List of branch names to merge into the target branch.") + args = parser.parse_args() - # Determine source branch early - source_branch = args.source - if not source_branch: - source_branch = get_current_branch() - if not source_branch: - sys.exit(1) + # Checkout the base branch to ensure a common starting point. + run_git_command(["git", "checkout", args.base]) - if not squash_and_merge(source_branch, args.target, args.title, args.backup, args.push): - sys.exit(2) + # Check if the target branch exists. If not, create it from the base branch. + branch_list = run_git_command(["git", "branch"], check=False) + branch_names = [line.strip("* ").strip() for line in branch_list.splitlines()] + if args.target in branch_names: + run_git_command(["git", "checkout", args.target]) + else: + run_git_command(["git", "checkout", "-b", args.target]) + + # Iterate over each branch, merging it with a squash merge. + for branch in args.branches: + print(f"Merging branch '{branch}' with a squash merge.") + # Merge the branch without creating a merge commit. + run_git_command(["git", "merge", "--squash", branch]) + # Commit the squashed changes with an appropriate message. + commit_message = args.title or f"Squashed merge of branch '{branch}'" + run_git_command(["git", "commit", "-m", commit_message]) + + print(f"All branches have been merged with squashed commits into '{args.target}'.") if __name__ == "__main__": diff --git a/release/ci/squash_and_merge_prs.py b/release/ci/squash_and_merge_prs.py index 259adcbb2e..5cd1b95746 100755 --- a/release/ci/squash_and_merge_prs.py +++ b/release/ci/squash_and_merge_prs.py @@ -80,7 +80,6 @@ def add_pr_comment(pr_number, comment): print(f"Failed to parse comments data for PR #{pr_number}") - def validate_pr(pr): """Validate a PR and return (is_valid, skip_reason)""" pr_number = pr.get('number', 'UNKNOWN') @@ -143,26 +142,30 @@ def process_pr(pr_data, source_branch, target_branch, squash_script_path): subprocess.run(['git', 'branch', branch, f'origin/{branch}'], check=True) # Run squash script - subprocess.run([ + result = subprocess.run([ squash_script_path, '--target', target_branch, - '--source', branch, + '--base', source_branch, '--title', f"{title} (PR-{pr_number})", - ], check=True) + branch, + ], capture_output=True, text=True) - print(f"Successfully processed PR #{pr_number}") - success_count += 1 + print(result.stdout) + if result.returncode == 0: + print(f"Successfully processed PR #{pr_number}") + success_count += 1 + continue - except subprocess.CalledProcessError as e: print(f"Error processing PR #{pr_number}:") - print(f"Command failed with exit code {e.returncode}") - error_output = getattr(e, 'stderr', 'No error output available') - print(f"Error output: {error_output}") - add_pr_comment(pr_number, - f"⚠️ Error during automated `{target_branch}` squash:\n```\n{error_output}\n```") + print(f"Command failed with exit code {result.returncode}") + output = result.stdout + print(f"Error output: {output}") + add_pr_comment(pr_number, f"⚠️ Error during automated `{target_branch}` squash:\n```\n{output}\n```") + subprocess.run(['git', 'reset', '--hard'], check=True) continue except Exception as e: print(f"Unexpected error processing PR #{pr_number}: {str(e)}") + subprocess.run(['git', 'reset', '--hard'], check=True) continue return success_count