Preserve original RELEASE skip accounting across shards

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "068a87c1fcc37d4f6a4bdec4c063cbf0189a8e439230eb7bb4e4659b0a39653d", "schema_version": 1}
+{"protected_inventory_sha256": "83e9cc537ef292cc228b0a7fcabd7828406cb4cad1c767b591bccdfc4c61f6fa", "schema_version": 1}

tools/ci/lifecycle_shards.py

@@ -167,6 +167,13 @@ def partition(targets, identities, count):
   return shards
 
 
+def release_result_allowed(record):
+  # Preserve the unsharded RELEASE runner's original unittest/pytest skip
+  # semantics. Skipped cases stay collected, source-bound and counted as
+  # skipped, never passed. Failures, errors and xfails remain unsuccessful.
+  return record['status'] in ('passed', 'skipped')
+
+
 def release_matches(identity, target):
   if identity.startswith('opendbc.'):
     identity = 'opendbc_repo.' + identity
@@ -214,6 +221,7 @@ def aggregate(directory, count, suite='aol-lifecycle'):
   targets = first['full_catalog_targets']
   expected = partitioner(targets, full, count)
   seen, indices = [], set()
+  passed_count = skipped_count = 0
   for doc in documents:
     index = doc['shard']['index']
     if index in indices or not 0 <= index < count or doc['shard']['count'] != count:
@@ -271,14 +279,19 @@ def aggregate(directory, count, suite='aol-lifecycle'):
     records = result['results']
     if result['exit_code'] != 0 or result['collection_errors'] or result['gate_errors']:
       raise ValueError('Lifecycle outcome report failed')
-    if result['selected_test_ids'] != ids or Counter(row['id'] for row in records) != Counter(ids) or any(row['status'] != 'passed' for row in records):
+    allowed = release_result_allowed if suite == 'safety-release' else lambda row: row['status'] == 'passed'
+    if result['selected_test_ids'] != ids or Counter(row['id'] for row in records) != Counter(ids) or any(not allowed(row) for row in records):
       raise ValueError('Lifecycle missing, duplicate or unsuccessful outcomes')
     if ids != [identity for identity in full if any(matcher(identity, target) for target in expected[index])]:
       raise ValueError('Lifecycle shard cases differ from the full catalog')
+    passed_count += sum(row['status'] == 'passed' for row in records)
+    skipped_count += sum(row['status'] == 'skipped' for row in records)
     seen.extend(ids)
   if Counter(seen) != Counter(full) or len(set(full)) != len(full):
     raise ValueError('Lifecycle aggregate does not cover every case exactly once')
-  return {'status': 'completed', 'shards': count, 'cases_passed': len(full), 'targets': len(targets), 'source_revision': first['source']['revision']}
+  return {'status': 'completed', 'shards': count, 'cases_passed': passed_count, 'cases_skipped': skipped_count,
+          'cases_total': len(full), 'targets': len(targets),
+          'source_revision': first['source']['revision']}
 
 
 def main():

tools/ci/run_vehicle_tests.py

@@ -814,7 +814,8 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4):
     gate_errors.extend(interface_execution_errors(platform_tests, records))
   if not collect_only and suite in ("aol-lifecycle", "safety-release"):
     gate_errors.extend(f"Required lifecycle case did not pass: {record['id']} ({record['status']})"
-                       for record in records if record["status"] != "passed")
+                       for record in records
+                       if not (lifecycle_shards.release_result_allowed(record) if suite == "safety-release" else record["status"] == "passed"))
     for target in plan["targets"]:
       if not any((lifecycle_shards.release_matches if suite == 'safety-release' else lifecycle_shards.matches)(identity, target) for identity in ids):
         gate_errors.append(f"Required lifecycle module collected no cases: {target}")

tools/ci/tests/test_lifecycle_shards.py

@@ -173,7 +173,7 @@ class LifecycleShardTests(unittest.TestCase):
   def test_release_aggregate_retains_all_outcome_and_native_guards(self):
     docs = self.documents('safety-release')
     self.assertEqual(self.check(copy.deepcopy(docs))['cases_passed'], 12)
-    for change in ('missing', 'duplicate', 'skip', 'debug', 'allOutput', 'wrong_suite'):
+    for change in ('missing', 'duplicate', 'debug', 'allOutput', 'wrong_suite'):
       bad = json.loads(json.dumps(docs))
       if change == 'missing':
         bad[0][1]['results'].pop()
@@ -190,6 +190,23 @@ class LifecycleShardTests(unittest.TestCase):
       with self.subTest(change=change), self.assertRaises((ValueError, KeyError, StopIteration)):
         self.check(bad)
 
+  def test_release_preserves_existing_skips_without_counting_them_as_passed(self):
+    docs = self.documents('safety-release')
+    docs[0][1]['results'][0].update(status='skipped', detail='DEBUG classic Bosch owner')
+    verdict = self.check(copy.deepcopy(docs))
+    self.assertEqual(verdict['cases_passed'], 11)
+    self.assertEqual(verdict['cases_skipped'], 1)
+    self.assertEqual(verdict['cases_total'], 12)
+    for status in ('failed', 'error', 'xfailed', 'xpassed', 'not_run', 'timed_out'):
+      bad = copy.deepcopy(docs)
+      bad[0][1]['results'][0]['status'] = status
+      with self.subTest(status=status), self.assertRaises(ValueError):
+        self.check(bad)
+    for doc, _ in docs:
+      doc['suite'] = 'aol-lifecycle'
+    with self.assertRaises(ValueError):
+      self.check(docs)
+
   def test_complete_catalog_passes(self):
     self.assertEqual(self.check(self.documents())['cases_passed'], 12)