Respect submitted time ranges in private judging audits

roadscore/tools/event_replay_audit.py

@@ -4,7 +4,7 @@ import json
 from pathlib import Path
 
 
-def audit(run):
+def audit(run, expected_duration=None):
     def read(name):
         p = run / name
         return json.loads(p.read_text()) if p.exists() else {}
@@ -34,6 +34,10 @@ def audit(run):
         'path': last_ui.get('nonempty_path_draws', 0) > 0,
         'lanes': last_ui.get('nonempty_lane_draws', 0) > 0,
     }
+    if expected_duration is not None:
+        if expected_duration <= 0:raise ValueError('Expected duration must be positive')
+        checks.pop('native_eof')
+        checks['submitted_range_complete'] = launch.get('end_reason') == 'requested duration' and abs(summary.get('audio_seconds', 0)-expected_duration) <= 1
     return {
         'checks': checks, 'instrumented_pass': all(checks.values()),
         'audio_seconds': summary.get('audio_seconds'), 'accepted_jobs': len(accepted),
@@ -49,7 +53,8 @@ def audit(run):
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument('run', type=Path)
+    parser.add_argument('--expected-duration', type=float)
     args = parser.parse_args()
-    result = audit(args.run)
+    result = audit(args.run, args.expected_duration)
     (args.run / 'event_replay_audit.json').write_text(json.dumps(result, indent=2))
     print(json.dumps(result, indent=2))

roadscore/tools/judging_run.py

@@ -25,6 +25,9 @@ def main():
     args = parser.parse_args()
     manifest = json.loads(args.manifest.read_text())
     row = next(r for r in manifest['submissions'] if r['label'] == args.label)
+    ranges = args.manifest.with_name('judging_ranges.private.json')
+    if ranges.exists():
+        row.update(json.loads(ranges.read_text()).get(args.label, {}))
     if row.get('range_resolution') == 'pending metadata':
         raise SystemExit('Resolve submitted range before launching this entry')
     out = ROOT / 'results/community_judging'
@@ -80,7 +83,10 @@ def main():
         record.update(phase='finished' if result.returncode == 0 else 'failed', exit_code=result.returncode,
                       run_paths=[str(r) for r in runs], ended_wall=time.time())
         for run in runs:
-            subprocess.run([PYTHON, str(ROOT / 'tools/event_replay_audit.py'), str(run)], stdout=subprocess.DEVNULL, check=True)
+            audit_args = [PYTHON, str(ROOT / 'tools/event_replay_audit.py'), str(run)]
+            if row.get('duration_seconds', 86400) < 86400:
+                audit_args += ['--expected-duration', str(row['duration_seconds'])]
+            subprocess.run(audit_args, stdout=subprocess.DEVNULL, check=True)
     except Exception as error:
         record.update(phase='failed', error_type=type(error).__name__, error=str(error), ended_wall=time.time())
         raise

roadscore/tools/test_event_replay_audit.py

@@ -6,6 +6,14 @@ from event_replay_audit import audit
 
 
 class AuditTests(unittest.TestCase):
+    def test_submitted_range_requires_complete_duration(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "launch.json").write_text(json.dumps({"end_reason": "requested duration"}))
+            (root / "summary.json").write_text(json.dumps({"audio_seconds": 50}))
+            self.assertFalse(audit(root, 100)["checks"]["submitted_range_complete"])
+            self.assertTrue(audit(root, 50)["checks"]["submitted_range_complete"])
+
     def test_missing_evidence_cannot_pass(self):
         with tempfile.TemporaryDirectory() as directory:
             self.assertFalse(audit(Path(directory))['instrumented_pass'])