diff --git a/roadscore/tools/event_replay_audit.py b/roadscore/tools/event_replay_audit.py index 32c51e1370..277abaaf0f 100644 --- a/roadscore/tools/event_replay_audit.py +++ b/roadscore/tools/event_replay_audit.py @@ -4,7 +4,7 @@ import json from pathlib import Path -def audit(run): +def audit(run, expected_duration=None): def read(name): p = run / name return json.loads(p.read_text()) if p.exists() else {} @@ -34,6 +34,10 @@ def audit(run): 'path': last_ui.get('nonempty_path_draws', 0) > 0, 'lanes': last_ui.get('nonempty_lane_draws', 0) > 0, } + if expected_duration is not None: + if expected_duration <= 0:raise ValueError('Expected duration must be positive') + checks.pop('native_eof') + checks['submitted_range_complete'] = launch.get('end_reason') == 'requested duration' and abs(summary.get('audio_seconds', 0)-expected_duration) <= 1 return { 'checks': checks, 'instrumented_pass': all(checks.values()), 'audio_seconds': summary.get('audio_seconds'), 'accepted_jobs': len(accepted), @@ -49,7 +53,8 @@ def audit(run): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('run', type=Path) + parser.add_argument('--expected-duration', type=float) args = parser.parse_args() - result = audit(args.run) + result = audit(args.run, args.expected_duration) (args.run / 'event_replay_audit.json').write_text(json.dumps(result, indent=2)) print(json.dumps(result, indent=2)) diff --git a/roadscore/tools/judging_run.py b/roadscore/tools/judging_run.py index 10a98b36dc..f841e7b91d 100644 --- a/roadscore/tools/judging_run.py +++ b/roadscore/tools/judging_run.py @@ -25,6 +25,9 @@ def main(): args = parser.parse_args() manifest = json.loads(args.manifest.read_text()) row = next(r for r in manifest['submissions'] if r['label'] == args.label) + ranges = args.manifest.with_name('judging_ranges.private.json') + if ranges.exists(): + row.update(json.loads(ranges.read_text()).get(args.label, {})) if row.get('range_resolution') == 'pending metadata': raise SystemExit('Resolve submitted range before launching this entry') out = ROOT / 'results/community_judging' @@ -80,7 +83,10 @@ def main(): record.update(phase='finished' if result.returncode == 0 else 'failed', exit_code=result.returncode, run_paths=[str(r) for r in runs], ended_wall=time.time()) for run in runs: - subprocess.run([PYTHON, str(ROOT / 'tools/event_replay_audit.py'), str(run)], stdout=subprocess.DEVNULL, check=True) + audit_args = [PYTHON, str(ROOT / 'tools/event_replay_audit.py'), str(run)] + if row.get('duration_seconds', 86400) < 86400: + audit_args += ['--expected-duration', str(row['duration_seconds'])] + subprocess.run(audit_args, stdout=subprocess.DEVNULL, check=True) except Exception as error: record.update(phase='failed', error_type=type(error).__name__, error=str(error), ended_wall=time.time()) raise diff --git a/roadscore/tools/test_event_replay_audit.py b/roadscore/tools/test_event_replay_audit.py index 10c623eff9..ffd079c43b 100644 --- a/roadscore/tools/test_event_replay_audit.py +++ b/roadscore/tools/test_event_replay_audit.py @@ -6,6 +6,14 @@ from event_replay_audit import audit class AuditTests(unittest.TestCase): + def test_submitted_range_requires_complete_duration(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "launch.json").write_text(json.dumps({"end_reason": "requested duration"})) + (root / "summary.json").write_text(json.dumps({"audio_seconds": 50})) + self.assertFalse(audit(root, 100)["checks"]["submitted_range_complete"]) + self.assertTrue(audit(root, 50)["checks"]["submitted_range_complete"]) + def test_missing_evidence_cannot_pass(self): with tempfile.TemporaryDirectory() as directory: self.assertFalse(audit(Path(directory))['instrumented_pass'])