From bf512d137c8eda8b2bc02ab1330ceba0b8ba678a Mon Sep 17 00:00:00 2001 From: "KOKO\\Mimi" Date: Wed, 14 Oct 2026 12:31:08 +0900 Subject: [PATCH] feat(performance): add single-run timing and optimized symbol builds --- CMakePresets.json | 13 ++ scripts/performance/measure.py | 306 ++++++++++++++++++++++++++++++ src/fesa/CMakeLists.txt | 9 + tests/test_performance_measure.py | 268 ++++++++++++++++++++++++++ 4 files changed, 596 insertions(+) create mode 100644 scripts/performance/measure.py create mode 100644 tests/test_performance_measure.py diff --git a/CMakePresets.json b/CMakePresets.json index 3e9acbc..fd20d76 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -13,12 +13,25 @@ "TBB_DIR": "$env{TBB_DIR}", "HDF5_DIR": "$env{HDF5_DIR}" } + }, + { + "name": "windows-msvc-ninja-release", + "displayName": "Windows MSVC x64 Release with symbols (Ninja)", + "inherits": "windows-msvc-ninja-debug", + "binaryDir": "${sourceDir}/.harness/build-ninja-msvc-release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } } ], "buildPresets": [ { "name": "windows-msvc-ninja-debug", "configurePreset": "windows-msvc-ninja-debug" + }, + { + "name": "windows-msvc-ninja-release", + "configurePreset": "windows-msvc-ninja-release" } ], "testPresets": [ diff --git a/scripts/performance/measure.py b/scripts/performance/measure.py new file mode 100644 index 0000000..09cad55 --- /dev/null +++ b/scripts/performance/measure.py @@ -0,0 +1,306 @@ +"""Measure whole native processes; reference comparison remains a separate step.""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import platform +import shutil +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[2] +CASES = ( + ("hexa_nonlinearstatic", ROOT / "reference/hexa/hexa_nonlinearstatic.inp", 2.0), + ("hexa_dynnonlinear", ROOT / "reference/hexa/hexa_dynnonlinear.inp", 100.0), + ("shellNL-dyn-meshfine", ROOT / "reference/shellNL-dyn-meshfine/shellNL-dyn-meshfine.inp", 100.0), +) +THREAD_KEYS = ("MKL_NUM_THREADS", "MKL_DYNAMIC", "MKL_THREADING_LAYER", "MKL_CBWR", + "TBB_NUM_THREADS", "OMP_NUM_THREADS", "OMP_DYNAMIC", "OMP_PROC_BIND", + "OMP_PLACES", "KMP_AFFINITY", "KMP_BLOCKTIME", "FESA_NUM_THREADS") + + +def sha256(path): + if not path.is_file(): + return None + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def publish(path, value): + """Expose only a fully serialized record; retain the last closed report.""" + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def command_output(command): + try: + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, + errors="replace", timeout=30, check=False) + return {"command": command, "exit": result.returncode, + "output": (result.stdout + result.stderr).strip()} + except (OSError, subprocess.TimeoutExpired) as error: + return {"command": command, "exit": None, "error": str(error)} + + +def collect_environment(exe): + """Capture actual build/runtime evidence outside every timed interval.""" + build = next((p for p in exe.parents if (p / "CMakeCache.txt").is_file()), None) + cache = (build / "CMakeCache.txt").read_text(errors="replace") if build else None + result = { + "source_commit": command_output(["git", "rev-parse", "HEAD"]), + "source_status": command_output(["git", "status", "--porcelain"]), + "measurement_tool_sha256": sha256(Path(__file__)), + "comparator_sha256": {name: sha256(ROOT / "scripts/verification" / name) + for name in ("compare_hex8.py", "compare_hht_dynamic.py")}, + "source_diff_sha256": hashlib.sha256(command_output(["git", "diff", "HEAD", "--", "include", "src", "cmake", "CMakeLists.txt", "CMakePresets.json"])["output"].encode()).hexdigest(), + "exe": str(exe), "exe_sha256": sha256(exe), + "os": platform.platform(), "cpu": platform.processor(), + "logical_processors": os.cpu_count(), + "thread_environment": {key: os.environ.get(key) for key in THREAD_KEYS}, + "related_environment": {key: value for key, value in sorted(os.environ.items()) + if key.startswith(("MKL_", "TBB_", "OMP_", "KMP_", "FESA_"))}, + "runtime_path": os.environ.get("PATH"), + "thread_settings_note": "Environment requests; TBB uses runtime default (TBB_NUM_THREADS is not an enforced cap). Actual thread use requires profile evidence.", + "build_type_flags_dependencies": cache, + "build_files": {str(p): sha256(p) for p in ((build / "CMakeCache.txt", build / "build.ninja") if build else ())}, + "runtime_dlls": {p.name: sha256(p) for p in sorted(exe.parent.glob("*.dll"))}, + "compiler": command_output(["cl"]), + "vtune": command_output([os.environ.get("VTUNE_EXE", "vtune"), "-version"]), + } + if os.name == "nt": + import ctypes + class PowerStatus(ctypes.Structure): + _fields_ = [("ac_line", ctypes.c_ubyte), ("battery_flag", ctypes.c_ubyte), + ("battery_percent", ctypes.c_ubyte), ("reserved", ctypes.c_ubyte), + ("battery_life", ctypes.c_ulong), ("battery_full_life", ctypes.c_ulong)] + power = PowerStatus() + obtained = bool(ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(power))) + result["system_power"] = {"obtained": obtained, + "ac_line_status": power.ac_line if obtained else None, + "battery_flag": power.battery_flag if obtained else None} + result["power_scheme"] = command_output(["powercfg", "/getactivescheme"]) + machine = command_output([ + "powershell", "-NoProfile", "-Command", + "ConvertTo-Json -Depth 4 @{boot=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o'); cpu=(Get-CimInstance Win32_Processor | Select-Object Name,NumberOfCores,NumberOfLogicalProcessors); battery=(Get-CimInstance Win32_Battery | Select-Object BatteryStatus); runtime=(Get-ChildItem -LiteralPath '" + + str(exe.parent).replace("'", "''") + + "' -Filter *.dll | ForEach-Object { @{name=$_.Name;version=$_.VersionInfo.FileVersion} })}"]) + if machine.get("exit") == 0: + machine["output"] = json.loads(machine["output"]) + result["machine_power_runtime_versions"] = machine + return result + + +def validate_result(case, scratch, result): + """Reuse established inventory/finite prechecks without claiming comparison.""" + name = "compare_hex8" if case.startswith("hexa_") else "compare_hht_dynamic" + spec = importlib.util.spec_from_file_location(name, ROOT / f"scripts/verification/{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if case.startswith("hexa_"): + model = module.input_inventory(scratch, case) + values = module.read_results(result, model) + module.check_result_values(values, model) + return {**values["evidence"], "physical_values_checked": True} + nodes, identity = module.input_inventory(scratch) + _, evidence = module.read_results(result, nodes, identity) + import h5py + with h5py.File(result, "r") as file: + if int(file["metadata"].attrs["schema_version"]) != 2: + raise ValueError("Shell performance requires schema v2") + source = file["metadata"].attrs["source_input_identity"] + if isinstance(source, bytes): + source = source.decode() + if source != f"path={scratch.absolute().as_posix()};content_identity={identity}": + raise ValueError("Shell result scratch path identity mismatch") + return evidence + + +def run_once(exe, original, case, directory, index, timeout=None): + directory.mkdir(parents=True, exist_ok=False) + scratch = directory / original.name + original_before = sha256(original) + shutil.copyfile(original, scratch) + scratch_before = sha256(scratch) + if original_before != scratch_before: + raise ValueError("Input copy differs from source bytes") + result = scratch.with_name(scratch.stem + "_results.h5") + native_path, stdout_path = directory / "native.json", directory / "stdout.txt" + command = [str(exe), str(scratch)] + state, native_exit, error, process = "completed", None, None, None + with stdout_path.open("wb") as output: + start = time.perf_counter() + try: + process = subprocess.Popen(command, cwd=directory, stdout=output, stderr=subprocess.STDOUT) + native_exit = process.wait(timeout=timeout) + except OSError as exception: + state, error = "launch-failed", str(exception) + except (KeyboardInterrupt, subprocess.TimeoutExpired) as exception: + state = "interrupted" if isinstance(exception, KeyboardInterrupt) else "timed-out" + error = str(exception) + if process is not None: + process.kill() + process.communicate() + native_exit = process.returncode + finally: + elapsed = time.perf_counter() - start + native = { + "case": case, "command": command, "cwd": str(directory), "native_exit": native_exit, + "execution_state": state, "original_sha256": original_before, + "scratch_sha256": scratch_before, "result_sha256": sha256(result), + "source_unchanged": sha256(original) == original_before, + "scratch_unchanged": sha256(scratch) == scratch_before, "result_exists": result.is_file(), + "original_sha256_after": sha256(original), "scratch_sha256_after": sha256(scratch), + "exe_sha256": sha256(exe), "elapsed_seconds": elapsed, + } + publish(native_path, native) + valid = (state == "completed" and native_exit == 0 and native["result_exists"] + and native["source_unchanged"] and native["scratch_unchanged"] + and math.isfinite(elapsed) and elapsed >= 0) + evidence, validation_state = None, "not-run" + if valid: + try: + evidence = validate_result(case, scratch, result) + validation_state = "passed" + except KeyboardInterrupt: + valid, validation_state = False, "interrupted" + error = "Result validation interrupted after native completion" + except Exception as exception: + valid, validation_state, error = False, "failed", str(exception) + row = {**native, "kind": "measurement", + "measurement_index": index, "original_input": str(original), "scratch_input": str(scratch), + "result": str(result), "native_record": str(native_path), "stdout_log": str(stdout_path), + "native_record_sha256": sha256(native_path), + "solver_log": str(scratch.with_name(scratch.stem + "_log.txt")), + "comparison_status": "pending", "comparison_report": None, + "execution_valid": valid, "result_precheck": evidence, + "result_validation_state": validation_state, "error": error} + publish(directory / "run.json", row) + return row + + +def measure(exe, output_dir, cases=CASES): + if output_dir.exists() and any(output_dir.iterdir()): + raise FileExistsError(f"Nonempty artifact directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + environment = collect_environment(exe) + report = {"schema": "fesa-performance-v1", "environment": environment, "cases": [], + "protocol": {"warmup_runs": 0, "measurement_runs": 1, + "cache_preparation": "no dedicated warmup or cache reset"}, + "execution_complete": False} + path = output_dir / "measurements.json" + publish(path, report) + for case, original, target in cases: + case_report = {"case": case, "target_seconds": target, "runs": [], + "all_measured_runs_within_target": False} + report["cases"].append(case_report) + if collect_environment(exe) != environment: + report["error"] = "Source, binary, build or environment changed during measurement" + publish(path, report) + return 1 + row = run_once(exe, original, case, output_dir / case / "run-1", 1) + native_valid = row["execution_valid"] + row["execution_valid"], row["within_target"] = False, False + row["environment_validation_state"] = "pending" + case_report["runs"].append(row) + publish(Path(row["native_record"]).parent / "run.json", row) + publish(path, report) + try: + unchanged = collect_environment(exe) == environment + except KeyboardInterrupt: + row["environment_validation_state"] = "interrupted" + row["error"] = "Environment validation interrupted after native completion" + except Exception as error: + row["environment_validation_state"] = "failed" + row["error"] = f"Environment validation failed after native completion: {error}" + else: + row["environment_validation_state"] = "passed" if unchanged else "changed" + row["execution_valid"] = native_valid and unchanged + row["within_target"] = row["execution_valid"] and row["elapsed_seconds"] <= target + if not unchanged: + row["error"] = "Source, binary, build or environment changed during native execution" + publish(Path(row["native_record"]).parent / "run.json", row) + case_report["all_measured_runs_within_target"] = row["within_target"] + publish(path, report) + print(f"{case} measurement 1: {row['elapsed_seconds']:.6f}s exit={row['native_exit']} valid={row['execution_valid']}", flush=True) + if not row["execution_valid"]: + return 1 + report["execution_complete"] = True + publish(path, report) + return 0 + + +def link_comparisons(output_dir): + """Attach closed comparator reports by hashes without changing raw timing.""" + path = output_dir / "measurements.json" + report = json.loads(path.read_text(encoding="utf-8")) + for case in report["cases"]: + for row in case["runs"]: + comparison_path = Path(row["native_record"]).parent / "comparison.json" + if not comparison_path.is_file(): + continue + comparison = json.loads(comparison_path.read_text(encoding="utf-8")) + identities = comparison.get("identities", {}) + native_path = Path(row["native_record"]) + native = json.loads(native_path.read_text(encoding="utf-8")) + if (sha256(native_path) != row["native_record_sha256"] + or row["elapsed_seconds"] != native["elapsed_seconds"] + or sha256(Path(row["scratch_input"])) != row["scratch_sha256"] + or sha256(Path(row["result"])) != row["result_sha256"] + or identities.get("input", {}).get("sha256") != row["scratch_sha256"] + or identities.get("results", {}).get("sha256") != row["result_sha256"] + or comparison.get("case") != case["case"]): + raise ValueError("Comparison does not identify the recorded native run/result") + if case["case"].startswith("hexa_") and comparison.get("blocking_passed") and ( + comparison.get("execution", {}).get("record_sha256") != row["native_record_sha256"]): + raise ValueError("Hex8 comparator did not verify this execution record") + row["comparison_status"] = "passed" if ( + row["execution_valid"] and comparison.get("precheck", {}).get("passed") + and comparison.get("blocking_passed") and comparison.get("verdict") == "passed" + ) else "failed" + row["comparison_report"] = str(comparison_path) + row["comparison_report_sha256"] = sha256(comparison_path) + report["comparisons_complete"] = (report["execution_complete"] and all( + row["comparison_status"] != "pending" for case in report["cases"] for row in case["runs"])) + report["all_comparisons_passed"] = (report["comparisons_complete"] and all( + row["comparison_status"] == "passed" for case in report["cases"] for row in case["runs"])) + publish(path, report) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--exe", type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--link-comparisons", action="store_true", + help="Attach existing per-run comparison.json by native/input/result hashes.") + args = parser.parse_args() + if args.link_comparisons: + try: + return link_comparisons(args.output_dir.resolve()) + except (OSError, ValueError, KeyError) as error: + print(f"comparison link failed: {error}", flush=True) + return 1 + if args.exe is None: + parser.error("--exe is required for measurement") + if not args.exe.is_file(): + parser.error("--exe must be an existing executable") + if args.output_dir.exists() and (not args.output_dir.is_dir() or any(args.output_dir.iterdir())): + parser.error("--output-dir must be new or empty") + try: + return measure(args.exe.resolve(), args.output_dir.resolve()) + except (OSError, ValueError, KeyboardInterrupt) as error: + print(f"measurement failed: {error}", flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fesa/CMakeLists.txt b/src/fesa/CMakeLists.txt index f254d81..77a683d 100644 --- a/src/fesa/CMakeLists.txt +++ b/src/fesa/CMakeLists.txt @@ -131,6 +131,15 @@ target_compile_options( $<$:/W4 /WX /EHsc> ) +# Profile the optimized production build with the same binary used for timing. +foreach(_fesa_target fesa_solver fesa_cli) + target_compile_options(${_fesa_target} PRIVATE + $<$,$>:/Zi>) +endforeach() +target_link_options(fesa_cli PRIVATE + $<$,$>:/DEBUG /OPT:REF /OPT:ICF>) +unset(_fesa_target) + if(WIN32) if(NOT TARGET MKL::mkl_intel_thread OR NOT TARGET MKL::mkl_core OR NOT OMP_DLL_DIR OR NOT OMP_DLLNAME) diff --git a/tests/test_performance_measure.py b/tests/test_performance_measure.py new file mode 100644 index 0000000..948b5b5 --- /dev/null +++ b/tests/test_performance_measure.py @@ -0,0 +1,268 @@ +"""Exercise process boundaries and preserved raw evidence, not solver numerics.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("measure", ROOT / "scripts/performance/measure.py") +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) + + +@pytest.fixture +def source(tmp_path): + path = tmp_path / "model.inp" + path.write_text("from pathlib import Path\nimport time\ntime.sleep(0.08)\n" + "Path(__file__).with_name('model_results.h5').write_bytes(b'fresh')\n") + return path + + +def test_real_spawn_isolated_bytes_and_pending_comparison(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {"frames": 101}) + before = source.read_bytes() + rows = [M.run_once(Path(sys.executable), source, "fixture", tmp_path / str(i), i) + for i in range(2)] + assert source.read_bytes() == before + for i, row in enumerate(rows): + assert row["native_exit"] == 0 and row["execution_state"] == "completed" + assert row["elapsed_seconds"] >= 0.08 + assert row["comparison_status"] == "pending" + assert row["measurement_index"] == i + native = json.loads(Path(row["native_record"]).read_text()) + assert len(native["command"]) == 2 + assert native["original_sha256"] == native["scratch_sha256"] + assert native["scratch_unchanged"] and native["source_unchanged"] + assert rows[0]["scratch_input"] != rows[1]["scratch_input"] + + +def test_timer_excludes_preparation_and_validation(tmp_path, source, monkeypatch): + events = [] + real_copy, real_spawn = M.shutil.copyfile, M.subprocess.Popen + monkeypatch.setattr(M.shutil, "copyfile", lambda *a: (events.append("copy"), real_copy(*a))[1]) + monkeypatch.setattr(M.time, "perf_counter", lambda: (events.append("clock"), float(len(events)))[1]) + monkeypatch.setattr(M.subprocess, "Popen", lambda *a, **kw: (events.append("spawn"), real_spawn(*a, **kw))[1]) + monkeypatch.setattr(M, "validate_result", lambda *a: (events.append("validate"), {})[1]) + M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert events == ["copy", "clock", "spawn", "clock", "validate"] + + +def test_nonzero_with_output_is_not_success(tmp_path, source, monkeypatch): + source.write_text(source.read_text() + "raise SystemExit(7)\n") + monkeypatch.setattr(M, "validate_result", lambda *a: pytest.fail("invalid native exit validated")) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert row["native_exit"] == 7 and not row["execution_valid"] + assert row["result_sha256"] is not None + + +def test_existing_directory_and_old_output_rejected(tmp_path, source): + run = tmp_path / "run" + run.mkdir() + old = run / "model_results.h5" + old.write_bytes(b"old") + with pytest.raises(FileExistsError): + M.run_once(Path(sys.executable), source, "fixture", run, 0) + assert old.read_bytes() == b"old" + + +def test_launch_failure_never_invents_result_hash(tmp_path, source): + row = M.run_once(tmp_path / "absent.exe", source, "fixture", tmp_path / "run", 0) + assert row["execution_state"] == "launch-failed" and row["native_exit"] is None + assert row["result_sha256"] is None and not row["execution_valid"] + + +def test_input_mutation_invalidates_result(tmp_path, source, monkeypatch): + source.write_text(source.read_text() + "Path(__file__).write_text('changed')\n") + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert not row["execution_valid"] + assert not json.loads(Path(row["native_record"]).read_text())["scratch_unchanged"] + + +def test_interrupted_run_and_prior_case_are_preserved(tmp_path, source, monkeypatch): + real = M.subprocess.Popen + calls = 0 + class Interrupted: + def __init__(self, *a, **kw): + self.proc = real(*a, **kw) + def wait(self, timeout=None): + raise KeyboardInterrupt + def kill(self): + self.proc.kill() + def communicate(self): + return self.proc.communicate() + @property + def returncode(self): + return self.proc.returncode + def spawn(*a, **kw): + nonlocal calls + calls += 1 + return real(*a, **kw) if calls == 1 else Interrupted(*a, **kw) + monkeypatch.setattr(M.subprocess, "Popen", spawn) + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) == 1 + report = json.loads((out / "measurements.json").read_text()) + assert len(report["cases"]) == 2 + assert report["cases"][0]["runs"][0]["execution_state"] == "completed" + assert report["cases"][1]["runs"][0]["execution_state"] == "interrupted" + assert not report["cases"][1]["all_measured_runs_within_target"] + + +def test_timeout_preserves_actual_exit(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: pytest.fail("timed out")) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0, timeout=0.001) + assert row["execution_state"] == "timed-out" + assert type(row["native_exit"]) is int and not row["execution_valid"] + + +def test_native_zero_missing_or_invalid_result_fails(tmp_path, source, monkeypatch): + source.write_text("pass\n") + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "missing", 0) + assert row["native_exit"] == 0 and not row["execution_valid"] + assert row["result_sha256"] is None + source.write_text("from pathlib import Path\nPath(__file__).with_name('model_results.h5').write_bytes(b'bad schema')\n") + def reject(*a): + raise ValueError("invalid inventory") + monkeypatch.setattr(M, "validate_result", reject) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "invalid", 0) + assert row["native_exit"] == 0 and not row["execution_valid"] + assert row["error"] == "invalid inventory" + + +def test_slow_runs_are_retained_and_exit_zero_is_not_time_pass(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 0.001)]) == 0 + report = json.loads((out / "measurements.json").read_text()) + case = report["cases"][0] + assert report["execution_complete"] and len(case["runs"]) == 1 + assert not case["all_measured_runs_within_target"] + assert case["runs"][0]["kind"] == "measurement" + assert case["runs"][0]["measurement_index"] == 1 + assert all(r["comparison_status"] == "pending" for r in case["runs"]) + + +def test_changed_binary_or_source_blocks_group(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + captures = iter([{"sha": "before"}, {"sha": "before"}, {"sha": "after"}]) + monkeypatch.setattr(M, "collect_environment", lambda *a: next(captures)) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 1)]) == 1 + row = json.loads((out / "measurements.json").read_text())["cases"][0]["runs"][0] + assert row["native_exit"] == 0 and not row["execution_valid"] + + +def test_cli_bad_arguments_are_exit_two(tmp_path): + proc = subprocess.run([sys.executable, str(ROOT / "scripts/performance/measure.py"), + "--exe", str(tmp_path / "missing.exe"), "--output-dir", str(tmp_path)], + capture_output=True) + assert proc.returncode == 2 + + +def test_comparison_link_preserves_raw_time_and_rejects_wrong_result(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 1)]) == 0 + before = json.loads((out / "measurements.json").read_text()) + row = before["cases"][0]["runs"][0] + comparison_path = Path(row["native_record"]).parent / "comparison.json" + comparison = {"case": "fixture", "identities": { + "input": {"sha256": row["scratch_sha256"]}, "results": {"sha256": row["result_sha256"]}}, + "precheck": {"passed": True}, "blocking_passed": True, "verdict": "passed"} + M.publish(comparison_path, comparison) + assert M.link_comparisons(out) == 0 + after = json.loads((out / "measurements.json").read_text()) + assert after["cases"][0]["runs"][0]["comparison_status"] == "passed" + assert [r["elapsed_seconds"] for r in after["cases"][0]["runs"]] == [ + r["elapsed_seconds"] for r in before["cases"][0]["runs"]] + comparison["identities"]["results"]["sha256"] = "other run" + M.publish(comparison_path, comparison) + with pytest.raises(ValueError): + M.link_comparisons(out) + assert json.loads((out / "measurements.json").read_text()) == after + + +def test_result_validation_interrupt_preserves_completed_native_and_prior_case(tmp_path, source, monkeypatch): + calls = 0 + def validate(*a): + nonlocal calls + calls += 1 + if calls == 2: + raise KeyboardInterrupt + return {} + monkeypatch.setattr(M, "validate_result", validate) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + try: + code = M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) + except KeyboardInterrupt: + pytest.fail("Result validation interrupt escaped without a completed native row") + assert code == 1 + report = json.loads((out / "measurements.json").read_text()) + assert len(report["cases"]) == 2 and not report["execution_complete"] + assert report["cases"][0]["runs"][0]["execution_valid"] + row = report["cases"][1]["runs"][0] + native = json.loads(Path(row["native_record"]).read_text()) + assert row["native_exit"] == native["native_exit"] == 0 + assert row["execution_state"] == native["execution_state"] == "completed" + assert row["elapsed_seconds"] == native["elapsed_seconds"] + assert row["result_sha256"] == native["result_sha256"] is not None + assert row["result_validation_state"] == "interrupted" and not row["execution_valid"] + assert row["comparison_status"] == "pending" + + +@pytest.mark.parametrize("exception,state", [(KeyboardInterrupt, "interrupted"), (OSError, "failed")]) +def test_post_native_environment_failure_preserves_current_and_prior_case(tmp_path, source, monkeypatch, exception, state): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + out = tmp_path / "artifacts" + def collect(*a): + if (out / "second/run-1/native.json").is_file(): + raise exception("post-native environment unavailable") + return {"fixed": True} + monkeypatch.setattr(M, "collect_environment", collect) + try: + code = M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) + except (KeyboardInterrupt, OSError): + pytest.fail("Post-native environment failure escaped without preserving the current row") + assert code == 1 + report = json.loads((out / "measurements.json").read_text()) + assert report["cases"][0]["runs"][0]["execution_valid"] + row = report["cases"][1]["runs"][0] + native = json.loads(Path(row["native_record"]).read_text()) + assert row["native_exit"] == native["native_exit"] == 0 + assert row["execution_state"] == native["execution_state"] == "completed" + assert row["elapsed_seconds"] == native["elapsed_seconds"] + assert row["result_sha256"] == native["result_sha256"] is not None + assert row["environment_validation_state"] == state + assert not row["execution_valid"] and row["error"] + assert not row["within_target"] and not report["execution_complete"] + assert json.loads((out / "second/run-1/run.json").read_text()) == row + + +@pytest.mark.parametrize("case,value,code", [ + ("hexa_nonlinearstatic", float("nan"), "nonfinite-result-value"), + ("hexa_nonlinearstatic", float("inf"), "nonfinite-result-value"), + ("hexa_dynnonlinear", 1e-12, "nonzero-initial-result-state"), +]) +def test_hex8_precheck_rejects_invalid_physical_values(tmp_path, case, value, code): + import h5py + helper_spec = importlib.util.spec_from_file_location("hex8_fixture", ROOT / "tests/test_compare_hex8.py") + helper = importlib.util.module_from_spec(helper_spec) + helper_spec.loader.exec_module(helper) + scratch = tmp_path / f"{case}.inp" + M.shutil.copyfile(ROOT / "reference/hexa" / scratch.name, scratch) + result = tmp_path / f"{case}_results.h5" + model = helper.COMPARE.input_inventory(scratch, case) + helper.make_result(result, model) + with h5py.File(result, "r+") as file: + file["steps/Step-1/frames/0/nodal/displacement"][0, 0] = value + with pytest.raises(Exception) as error: + M.validate_result(case, scratch, result) + assert error.value.code == code