diff --git a/CMakePresets.json b/CMakePresets.json index 3e9acbc..fd20d76 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -13,12 +13,25 @@ "TBB_DIR": "$env{TBB_DIR}", "HDF5_DIR": "$env{HDF5_DIR}" } + }, + { + "name": "windows-msvc-ninja-release", + "displayName": "Windows MSVC x64 Release with symbols (Ninja)", + "inherits": "windows-msvc-ninja-debug", + "binaryDir": "${sourceDir}/.harness/build-ninja-msvc-release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } } ], "buildPresets": [ { "name": "windows-msvc-ninja-debug", "configurePreset": "windows-msvc-ninja-debug" + }, + { + "name": "windows-msvc-ninja-release", + "configurePreset": "windows-msvc-ninja-release" } ], "testPresets": [ diff --git a/docs/calculation-performance/verification.md b/docs/calculation-performance/verification.md new file mode 100644 index 0000000..369165c --- /dev/null +++ b/docs/calculation-performance/verification.md @@ -0,0 +1,210 @@ +# 계산 성능 기준 측정과 도구 검증 + +T1은 production 계산 코드를 변경하지 않고 Release 설정, 외부 측정기와 +기준 실행·VTune 자료를 준비한다. 최종 전체 Debug 회귀는 T5 책임이다. +기준 실행은 승인된 계산 소스 `eaa5cbeb21b231862ed80f89386412232b0571c9`와 +동일한 production 파일로 수행한다. 시간 목표 초과는 기준자료의 미달로 기록한다. + +## 재현 명령과 소유권 + +MSVC x64 Developer PowerShell에서 기존 로컬 GoogleTest/MKL/TBB/HDF5 설정을 +사용한다. 공용 source에는 개인 경로나 자동 다운로드를 추가하지 않는다. + +```powershell +cmake --preset windows-msvc-ninja-release +cmake --build --preset windows-msvc-ninja-release --target fesa_cli --parallel 8 +python scripts/performance/measure.py --exe .harness/build-ninja-msvc-release/src/fesa/fesa.exe --output-dir <새 artifact 디렉터리> +``` + +Release는 `/O2 /Ob2 /DNDEBUG /Zi`, link는 `/DEBUG /OPT:REF /OPT:ICF`다. +기존 Debug preset과 개인 config를 유지한다. 측정기만 단조 증가 시계와 native +프로세스·raw 실행 기록을 소유한다. 각 입력을 독립 디렉터리에 byte-copy하고 +프로세스 시작 직전부터 종료 확인 직후까지 측정한다. 복사와 hash, HDF5 inventory, +외부 비교는 타이머 밖이다. timeout은 공식 CLI에서 설정하지 않는다. + +2026-10-14 사용자 승인 PERF-ADR-004에 따라 워밍업 없이 모델별 1회만 측정한다. +각 새 결과의 identity·전체 frame·finite precheck를 +기존 comparator의 reader로 검사한다. Hex8은 기존 schema/row 검사를, shell은 +729 nodes/101 frames와 schema v2 및 실제 scratch 경로를 검사한다. +이 검사는 외부 reference 수치 비교의 통과를 의미하지 않는다. +실행 실패·중단·환경/source/binary 변경은 합격을 차단하고 완료된 raw 기록을 남긴다. + +각 run의 `native.json`·`run.json`·로그·새 HDF5를 보존한다. +`measurements.json`은 각 행이 완료된 뒤 atomic replace하며 중단된 case나 미시작 case를 +완료 표본으로 채우지 않는다. 성공한 느린 실행도 제외하지 않는다. +측정기 exit 0은 정상 전체 실행·기록 완료, exit 1은 실행/기록 실패, +exit 2는 잘못된 CLI다. exit 0은 시간 목표나 외부 비교의 합격이 아니다. + +비교는 각 실제 scratch/result/native로 별도 실행하며 `comparison.json`을 run +디렉터리에 둔다. Hex8 명령은 다음 형태다. + +```powershell +python scripts/verification/compare_hex8.py --case --input --results --references reference/hexa --execution-record --report +python scripts/verification/compare_hht_dynamic.py --input --results --reference "reference/shellNL-dyn-meshfine/shellNL-dyn-meshfine displacements.csv" --report +python scripts/performance/measure.py --output-dir --link-comparisons +``` + +연결 단계는 input/result/native SHA-256과 raw 경과 시간을 확인한다. +Hex8 합격에는 비교기가 검증한 동일 native record hash가 필요하다. +Shell의 U blocking/UR warning 정책은 기존 비교기를 그대로 따른다. +연결 단계는 raw 시간과 native 기록을 변경하지 않는다. 비교 전 상태는 `pending`이다. + +## 행동 검사 + +`uv run --offline --with pytest --with numpy --with h5py python -B -m pytest -q tests/test_performance_measure.py` + +측정기 구현 전 같은 명령 exit 1(`measure.py` 부재)을 확인했다. +실제 subprocess와 독립 관측으로 타이머 경계·copy byte 보존·실행 분리·기존 +출력 충돌·native nonzero/출력 존재·launch failure·입력 mutation·중단/timeout 기록, +native 0의 missing/invalid HDF5, 느린 표본 보존과 source/binary 변경 차단을 검사한다. +비교 연결은 raw 시간 보존 및 다른 result hash 거부도 검사한다. +단일 측정 변경과 독립 검토 수정 전에는 6 FAIL/11 PASS, exit 1을 확인했다. +현재 관련 검사는 19 PASS, exit 0이다. 실제 Hex8 HDF5의 NaN/Inf 및 0이 아닌 +동적 초기 변위를 기존 `check_result_values`로 거부한다. native 종료 뒤 HDF5 +검사 중 `KeyboardInterrupt`는 native `completed`/exit 0/경과 시간/hash를 보존하고 +`result_validation_state=interrupted`, `execution_valid=false` 행을 공개한 뒤 exit 1로 +종료한다. 이전 case의 완료 행도 남는다. +native 행 반환 뒤 환경 재검사 중단/실패도 별도로 2 FAIL, exit 1로 재현했다. +환경 재검사 전에 현재 행을 먼저 공개하고, `environment_validation_state`가 +`interrupted|failed`인 행의 실제 native/elapsed/hash를 유지해 exit 1로 종료한다. + +## 현재 기준 자료 + +새 공식 단일 기준 자료 위치는 repository root의 +`.harness/performance/baseline/t1-single-20261014-122548/`다. +MSVC 19.51.36252 x64, MKL 2026.1.0, TBB 2023.1, HDF5 2.1.1, +VTune 2026.2.0 build 632324를 사용한다. 전원·CPU/OS·실제 cache/flags, +dependency DLL hash/version, source commit/dirty, thread 요청은 JSON에 기록한다. +MKL/OMP thread 요청은 8, dynamic FALSE다. TBB는 runtime default이며 +`TBB_NUM_THREADS`를 실제 적용된 cap으로 주장하지 않는다. + +설치된 VTune help를 보존했으며 software sampling은 10ms fixed다. +`-collect hotspots -knob sampling-mode=sw -knob enable-stack-collection=true +-knob enable-characterization-insights=false -result-dir -- `를 +사용한다. characterization 하드웨어 카운터는 끄고 제품 PDB와 동일 Release binary로 +수집한다. 정적은 full, 두 dynamic은 `-duration 30`의 partial 수집이다. +부분 수집은 전체 시간 합격 표본이 아니며 collection 종료의 native/tool 상태를 +full 실행과 구분한다. raw 결과와 function/callstack CSV를 보존한다. + +VTune raw/function/callstack 자료는 같은 Release binary의 재부팅 전 자료 +`.harness/performance/baseline/t1-20261014-0845/profiles/`에 보존한다. +주요 product 함수와 source는 식별됐다. 일부 vendor 함수는 주소로만 식별되며 +해당 vendor의 private symbol은 확보하지 않았다. 이 한계와 partial 수집을 구분한다. + +## 이전 프로토콜과 재부팅 이력 + +사용자는 장시간 작업 중 PC 중단을 알리고 단일 측정으로 변경했다. 원인은 입증되지 +않았다. 실제 OS `LastBootUpTime`은 `2026-10-14T09:33:20.5000000+09:00`다. +복구 시 fesa/VTune/build/측정 프로세스 조회는 비어 있었다. +Win32 전원 조회는 AC line 1, battery flag 128이었다. + +과거 `.harness/performance/baseline/t1-20261014-0845/measurements/measurements.json`은 +static 워밍업+측정 5개, dynamic 워밍업+측정 4개 완료 행을 보존한다. +dynamic 측정 5의 native 종료·경과 시간은 관측되지 않았으며 native record와 +최종 HDF5가 없다. Shell은 시작하지 않았다. 해당 run의 scratch와 기존 raw 기록을 +보존하며 부재·현재 hash·재부팅·프로세스 조회를 `recovery-20261014.json`에 기록했다. +과거 결과는 공식 단일 판정에 사용하지 않는다. + +과거 frozen reader는 Hex8 schema/identity/frame을 검사했으나 독립 검토에서 +별도 `check_result_values` 누락과 HDF5 검사 중 중단 행 누락이 발견됐다. +두 문제는 위 행동 검사로 재현·수정했다. 과거 elapsed/native 기록을 수정하지 않았고 +frozen script SHA-256 `34772002e774efe566b5331641285a5dadb2194b10e2c365f90a561ba521ef04`도 +`build/measure-frozen-baseline.py`로 보존했다. 새 공식 결과는 수정된 검사와 전체 +기존 comparator를 적용한다. + +## 공식 단일 기준 실행 상태 + +독립 conformance 재검토 통과 뒤 준비 commit +`bf512d137c8eda8b2bc02ab1330ceba0b8ba678a`에서 모델별 1회만 실행했다. +워밍업과 별도 캐시 초기화를 수행하지 않았다. 측정 중 code/doc/gitstatus, +binary/DLL/thread/power 환경을 고정했다. 측정기 exit 0이며 각 solver도 실제 exit 0이다. +source/schema/frame/finite/초기 상태 및 환경 재검사를 통과했고, 기존 전체 comparator를 +타이머 밖에서 실행했다. 비교 연결 exit 0이며 raw elapsed/native 기록을 수정하지 않았다. + +| 모델 | 목표 초 | 전체 wall 초 | native | 외부 비교 | 시간 판정 | +| --- | ---: | ---: | ---: | --- | --- | +| hexa_nonlinearstatic | 2 | 10.914924 | 0 | 전체 U/constrained RF/point S 통과 | 미달 | +| hexa_dynnonlinear | 100 | 402.069447 | 0 | 101 frames 전체 U 통과 | 미달 | +| shellNL-dyn-meshfine | 100 | 6372.325621 | 0 | 729 nodes/101 frames 확인; 9 nodes U blocking 통과, UR warning 200 entries | 미달 | + +세 comparator의 실제 exit는 각각 0이며 reference/결과의 identity와 finite 검사 뒤 +기존 row 상대 허용 0.05, relative RMS 0.01, near-zero ratio 0.01을 적용했다. +최대 blocking relative RMS는 정적 `9.338900e-6`, 동적 `2.155586e-6`, shell U +`0.0022824245`다. UR warning은 기존 정책의 경고이며 회전 정확성 통과로 표시하지 않는다. +기준 자료 확보와 수치 비교는 완료됐으나 시간 목표는 세 모델 모두 실패다. +baseline 시간 목표는 T1의 진행 차단 조건이 아니며 최적화/최종 전체 검증은 후속 작업이다. + +실제 판정과 run/input/binary/result/native/comparison hash는 +[공식 measurements.json](../../.harness/performance/baseline/t1-single-20261014-122548/measurements/measurements.json)에 +연결됐다. `execution_complete`, `comparisons_complete`, `all_comparisons_passed`는 모두 true이고 +각 `all_measured_runs_within_target`은 false다. `run.json`/`native.json`은 실행 당시 raw 기록, +외부 비교의 현재 연결 상태는 최종 aggregate의 각 행이 소유한다. + +- [정적 전체 비교](../../.harness/performance/baseline/t1-single-20261014-122548/measurements/hexa_nonlinearstatic/run-1/comparison.json) +- [동적 전체 비교](../../.harness/performance/baseline/t1-single-20261014-122548/measurements/hexa_dynnonlinear/run-1/comparison.json) +- [Shell 전체 비교](../../.harness/performance/baseline/t1-single-20261014-122548/measurements/shellNL-dyn-meshfine/run-1/comparison.json) +- [실제 comparator 명령·종료·hash](../../.harness/performance/baseline/t1-single-20261014-122548/comparisons-execution.json) + +## VTune 비용과 후속 owner에 넘길 범위 + +재부팅 전 같은 binary의 정적 full profile은 `TwoSum` 1.648 CPU 초, +`Multiply` 1.055 CPU 초, `LowestBitExponent` 0.615 CPU 초를 관측했다. +동적 Hex8 30초 partial은 각각 37.424, 17.042, 22.038 CPU 초와 `ldexp` 19.263 CPU 초를 +관측했다. high+low/정밀도 의미를 변경할 근거로 사용하지 않는다. + +- [Hex8 정적 함수](../../.harness/performance/baseline/t1-20261014-0845/profiles/hexa_nonlinearstatic-functions.csv) / [호출 stack](../../.harness/performance/baseline/t1-20261014-0845/profiles/hexa_nonlinearstatic-callstacks.csv) +- [Hex8 동적 partial 함수](../../.harness/performance/baseline/t1-20261014-0845/profiles/hexa_dynnonlinear-partial-functions.csv) / [호출 stack](../../.harness/performance/baseline/t1-20261014-0845/profiles/hexa_dynnonlinear-partial-callstacks.csv) +- [최초 Shell partial 함수](../../.harness/performance/baseline/t1-20261014-0845/profiles/shellNL-dyn-meshfine-partial-functions.csv) / [호출 stack](../../.harness/performance/baseline/t1-20261014-0845/profiles/shellNL-dyn-meshfine-partial-callstacks.csv) +- [재부팅 전 collector 상태](../../.harness/performance/baseline/t1-20261014-0845/profiles/collection-execution.json) + +최초 shell 30초 로그는 initial increment 0만 포함했다. 공식 시간 측정과 3개 비교 종료 뒤 +동일 binary/source/thread 설정에서 sw 150초 partial을 1회 추가했다. 실제 accepted +increment 2/time 0.2까지 포함하며 수집 종료 후 target/collector 프로세스가 없었다. +VTune collector의 실제 native exit는 4, 함수/stack report의 실제 exit는 각각 0이다. +target 자체의 종료 코드는 별도로 관측하지 않아 null이며 정상 전체 해석 표본이 아니다. + +| 150초 partial의 함수/모듈 | sampled CPU 초 | 전체 sampled CPU 비중 | +| --- | ---: | ---: | +| shell Jet operator* | 133.466 | 27.6% | +| libiomp5md 주소 함수 | 70.081 | 14.5% | +| malloc_base | 30.289 | 6.3% | +| MKL PARDISO blkl_omp_mic_vbsr_real | 27.669 | 5.7% | +| inertia Jet operator* | 26.257 | 5.4% | + +T2 math/linear owner에는 `CheckGraphRegularity` → `CheckSelectorTransversality` → +`MklPardisoTangentSolver::FactorizeFresh/SolveCorrection` → `pardiso` 경로를 넘긴다. +leaf call-path의 양수 CPU weight를 caller별 한 번 합산한 보조 분석은 graph regularity +2.434118 CPU 초, nested selector 1.600014 CPU 초, PARDISO 42.725655 CPU 초다. +flat 함수별 CPU와 call-path weight의 합계가 일치한다. 이는 중첩 inclusive attribution이며 +서로 합산하거나 wall time으로 바꾸지 않는다. 현재 표본에서 graph/selector가 주된 +병목이라고 단정할 근거는 없다. + +T4 element/assembly owner에는 shell/inertia Jet 곱셈·할당과 +`EvaluateInternalResidualAndTangent`/`EvaluateAssembly`의 TBB 경로, allocator 및 MKL/OMP +소비 자료를 넘긴다. `EvaluateAssembly` caller의 보조 inclusive CPU는 257.862958 초다. +아직 수치 구현을 변경하거나 향후 개선 배수를 추정하지 않았다. + +- [150초 raw 결과](../../.harness/performance/baseline/t1-single-20261014-122548/profiles/shellNL-dyn-meshfine-repeat-150s) +- [함수 CSV](../../.harness/performance/baseline/t1-single-20261014-122548/profiles/shellNL-dyn-meshfine-repeat-150s-functions.csv) / [전체 stack CSV](../../.harness/performance/baseline/t1-single-20261014-122548/profiles/shellNL-dyn-meshfine-repeat-150s-callstacks.csv) +- [호출 경로별 보조 합계·방법](../../.harness/performance/baseline/t1-single-20261014-122548/profiles/callpath-attribution.json) +- [실제 partial 명령·exit 4·child 부재](../../.harness/performance/baseline/t1-single-20261014-122548/profiles/collection-execution.json) + +## 보존과 검증 범위 + +실행 binary SHA-256은 `fdb77bc39cc518d509e6bcf679d91f75d67a8c0bbde6b505fe555030274eccfb`, +공식 측정기 hash는 `4ae3dfcee43cd9aa25ce11a1a1c6d87646f745b881fac9ad6c79a52e1bc5b082`다. +실제 exe/PDB/DLL은 parent artifact의 `build/binary/`에 복사했고 공식 환경의 모든 runtime +DLL hash와 비교해 일치를 확인했다. 살아 있던 공식 Shell의 실제 DLL module 경로도 +모두 Release exe 옆 staged DLL이었다. source numerical diff는 승인 baseline 대비 비어 있다. + +- [Binary/PDB/DLL archive manifest](../../.harness/performance/baseline/t1-single-20261014-122548/build/binary-manifest.json) +- [빌드 명령 로그](../../.harness/performance/baseline/t1-single-20261014-122548/build/build-release.txt) / [configure 로그](../../.harness/performance/baseline/t1-single-20261014-122548/build/configure-release.txt) +- [최종 관련 Python 19 PASS](../../.harness/performance/baseline/t1-single-20261014-122548/build/pytest-single-green.txt) / [단일·finite·검증중단 RED](../../.harness/performance/baseline/t1-single-20261014-122548/build/pytest-single-red.txt) / [환경중단 RED](../../.harness/performance/baseline/t1-single-20261014-122548/build/pytest-environment-red.txt) +- [재부팅·전원·source/flags/version 환경](../../.harness/performance/baseline/t1-single-20261014-122548/build/environment-ready.json) +- [실제 DLL/보조 resource/프로세스 부재 관측](../../.harness/performance/baseline/t1-single-20261014-122548/build/live-observations.json) +- [과거 중단 복구 기록](../../.harness/performance/baseline/t1-20261014-0845/recovery-20261014.json) + +Release build 범위는 `fesa_cli`다. Release CTest discovery exit 0의 3개 항목은 모두 +`*_NOT_BUILT` placeholder이며 검사 통과가 아니다. 전체 MSVC x64 Debug build/CTest는 +T1에서 실행하지 않았으며 후속 T5의 필수 검증으로 남는다. 원본 reference, 물리·질량· +high+low·수치 tolerance와 결과 schema를 변경하지 않았다. diff --git a/scripts/performance/measure.py b/scripts/performance/measure.py new file mode 100644 index 0000000..09cad55 --- /dev/null +++ b/scripts/performance/measure.py @@ -0,0 +1,306 @@ +"""Measure whole native processes; reference comparison remains a separate step.""" +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import platform +import shutil +import subprocess +import time + +ROOT = Path(__file__).resolve().parents[2] +CASES = ( + ("hexa_nonlinearstatic", ROOT / "reference/hexa/hexa_nonlinearstatic.inp", 2.0), + ("hexa_dynnonlinear", ROOT / "reference/hexa/hexa_dynnonlinear.inp", 100.0), + ("shellNL-dyn-meshfine", ROOT / "reference/shellNL-dyn-meshfine/shellNL-dyn-meshfine.inp", 100.0), +) +THREAD_KEYS = ("MKL_NUM_THREADS", "MKL_DYNAMIC", "MKL_THREADING_LAYER", "MKL_CBWR", + "TBB_NUM_THREADS", "OMP_NUM_THREADS", "OMP_DYNAMIC", "OMP_PROC_BIND", + "OMP_PLACES", "KMP_AFFINITY", "KMP_BLOCKTIME", "FESA_NUM_THREADS") + + +def sha256(path): + if not path.is_file(): + return None + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def publish(path, value): + """Expose only a fully serialized record; retain the last closed report.""" + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def command_output(command): + try: + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, + errors="replace", timeout=30, check=False) + return {"command": command, "exit": result.returncode, + "output": (result.stdout + result.stderr).strip()} + except (OSError, subprocess.TimeoutExpired) as error: + return {"command": command, "exit": None, "error": str(error)} + + +def collect_environment(exe): + """Capture actual build/runtime evidence outside every timed interval.""" + build = next((p for p in exe.parents if (p / "CMakeCache.txt").is_file()), None) + cache = (build / "CMakeCache.txt").read_text(errors="replace") if build else None + result = { + "source_commit": command_output(["git", "rev-parse", "HEAD"]), + "source_status": command_output(["git", "status", "--porcelain"]), + "measurement_tool_sha256": sha256(Path(__file__)), + "comparator_sha256": {name: sha256(ROOT / "scripts/verification" / name) + for name in ("compare_hex8.py", "compare_hht_dynamic.py")}, + "source_diff_sha256": hashlib.sha256(command_output(["git", "diff", "HEAD", "--", "include", "src", "cmake", "CMakeLists.txt", "CMakePresets.json"])["output"].encode()).hexdigest(), + "exe": str(exe), "exe_sha256": sha256(exe), + "os": platform.platform(), "cpu": platform.processor(), + "logical_processors": os.cpu_count(), + "thread_environment": {key: os.environ.get(key) for key in THREAD_KEYS}, + "related_environment": {key: value for key, value in sorted(os.environ.items()) + if key.startswith(("MKL_", "TBB_", "OMP_", "KMP_", "FESA_"))}, + "runtime_path": os.environ.get("PATH"), + "thread_settings_note": "Environment requests; TBB uses runtime default (TBB_NUM_THREADS is not an enforced cap). Actual thread use requires profile evidence.", + "build_type_flags_dependencies": cache, + "build_files": {str(p): sha256(p) for p in ((build / "CMakeCache.txt", build / "build.ninja") if build else ())}, + "runtime_dlls": {p.name: sha256(p) for p in sorted(exe.parent.glob("*.dll"))}, + "compiler": command_output(["cl"]), + "vtune": command_output([os.environ.get("VTUNE_EXE", "vtune"), "-version"]), + } + if os.name == "nt": + import ctypes + class PowerStatus(ctypes.Structure): + _fields_ = [("ac_line", ctypes.c_ubyte), ("battery_flag", ctypes.c_ubyte), + ("battery_percent", ctypes.c_ubyte), ("reserved", ctypes.c_ubyte), + ("battery_life", ctypes.c_ulong), ("battery_full_life", ctypes.c_ulong)] + power = PowerStatus() + obtained = bool(ctypes.windll.kernel32.GetSystemPowerStatus(ctypes.byref(power))) + result["system_power"] = {"obtained": obtained, + "ac_line_status": power.ac_line if obtained else None, + "battery_flag": power.battery_flag if obtained else None} + result["power_scheme"] = command_output(["powercfg", "/getactivescheme"]) + machine = command_output([ + "powershell", "-NoProfile", "-Command", + "ConvertTo-Json -Depth 4 @{boot=(Get-CimInstance Win32_OperatingSystem).LastBootUpTime.ToString('o'); cpu=(Get-CimInstance Win32_Processor | Select-Object Name,NumberOfCores,NumberOfLogicalProcessors); battery=(Get-CimInstance Win32_Battery | Select-Object BatteryStatus); runtime=(Get-ChildItem -LiteralPath '" + + str(exe.parent).replace("'", "''") + + "' -Filter *.dll | ForEach-Object { @{name=$_.Name;version=$_.VersionInfo.FileVersion} })}"]) + if machine.get("exit") == 0: + machine["output"] = json.loads(machine["output"]) + result["machine_power_runtime_versions"] = machine + return result + + +def validate_result(case, scratch, result): + """Reuse established inventory/finite prechecks without claiming comparison.""" + name = "compare_hex8" if case.startswith("hexa_") else "compare_hht_dynamic" + spec = importlib.util.spec_from_file_location(name, ROOT / f"scripts/verification/{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if case.startswith("hexa_"): + model = module.input_inventory(scratch, case) + values = module.read_results(result, model) + module.check_result_values(values, model) + return {**values["evidence"], "physical_values_checked": True} + nodes, identity = module.input_inventory(scratch) + _, evidence = module.read_results(result, nodes, identity) + import h5py + with h5py.File(result, "r") as file: + if int(file["metadata"].attrs["schema_version"]) != 2: + raise ValueError("Shell performance requires schema v2") + source = file["metadata"].attrs["source_input_identity"] + if isinstance(source, bytes): + source = source.decode() + if source != f"path={scratch.absolute().as_posix()};content_identity={identity}": + raise ValueError("Shell result scratch path identity mismatch") + return evidence + + +def run_once(exe, original, case, directory, index, timeout=None): + directory.mkdir(parents=True, exist_ok=False) + scratch = directory / original.name + original_before = sha256(original) + shutil.copyfile(original, scratch) + scratch_before = sha256(scratch) + if original_before != scratch_before: + raise ValueError("Input copy differs from source bytes") + result = scratch.with_name(scratch.stem + "_results.h5") + native_path, stdout_path = directory / "native.json", directory / "stdout.txt" + command = [str(exe), str(scratch)] + state, native_exit, error, process = "completed", None, None, None + with stdout_path.open("wb") as output: + start = time.perf_counter() + try: + process = subprocess.Popen(command, cwd=directory, stdout=output, stderr=subprocess.STDOUT) + native_exit = process.wait(timeout=timeout) + except OSError as exception: + state, error = "launch-failed", str(exception) + except (KeyboardInterrupt, subprocess.TimeoutExpired) as exception: + state = "interrupted" if isinstance(exception, KeyboardInterrupt) else "timed-out" + error = str(exception) + if process is not None: + process.kill() + process.communicate() + native_exit = process.returncode + finally: + elapsed = time.perf_counter() - start + native = { + "case": case, "command": command, "cwd": str(directory), "native_exit": native_exit, + "execution_state": state, "original_sha256": original_before, + "scratch_sha256": scratch_before, "result_sha256": sha256(result), + "source_unchanged": sha256(original) == original_before, + "scratch_unchanged": sha256(scratch) == scratch_before, "result_exists": result.is_file(), + "original_sha256_after": sha256(original), "scratch_sha256_after": sha256(scratch), + "exe_sha256": sha256(exe), "elapsed_seconds": elapsed, + } + publish(native_path, native) + valid = (state == "completed" and native_exit == 0 and native["result_exists"] + and native["source_unchanged"] and native["scratch_unchanged"] + and math.isfinite(elapsed) and elapsed >= 0) + evidence, validation_state = None, "not-run" + if valid: + try: + evidence = validate_result(case, scratch, result) + validation_state = "passed" + except KeyboardInterrupt: + valid, validation_state = False, "interrupted" + error = "Result validation interrupted after native completion" + except Exception as exception: + valid, validation_state, error = False, "failed", str(exception) + row = {**native, "kind": "measurement", + "measurement_index": index, "original_input": str(original), "scratch_input": str(scratch), + "result": str(result), "native_record": str(native_path), "stdout_log": str(stdout_path), + "native_record_sha256": sha256(native_path), + "solver_log": str(scratch.with_name(scratch.stem + "_log.txt")), + "comparison_status": "pending", "comparison_report": None, + "execution_valid": valid, "result_precheck": evidence, + "result_validation_state": validation_state, "error": error} + publish(directory / "run.json", row) + return row + + +def measure(exe, output_dir, cases=CASES): + if output_dir.exists() and any(output_dir.iterdir()): + raise FileExistsError(f"Nonempty artifact directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + environment = collect_environment(exe) + report = {"schema": "fesa-performance-v1", "environment": environment, "cases": [], + "protocol": {"warmup_runs": 0, "measurement_runs": 1, + "cache_preparation": "no dedicated warmup or cache reset"}, + "execution_complete": False} + path = output_dir / "measurements.json" + publish(path, report) + for case, original, target in cases: + case_report = {"case": case, "target_seconds": target, "runs": [], + "all_measured_runs_within_target": False} + report["cases"].append(case_report) + if collect_environment(exe) != environment: + report["error"] = "Source, binary, build or environment changed during measurement" + publish(path, report) + return 1 + row = run_once(exe, original, case, output_dir / case / "run-1", 1) + native_valid = row["execution_valid"] + row["execution_valid"], row["within_target"] = False, False + row["environment_validation_state"] = "pending" + case_report["runs"].append(row) + publish(Path(row["native_record"]).parent / "run.json", row) + publish(path, report) + try: + unchanged = collect_environment(exe) == environment + except KeyboardInterrupt: + row["environment_validation_state"] = "interrupted" + row["error"] = "Environment validation interrupted after native completion" + except Exception as error: + row["environment_validation_state"] = "failed" + row["error"] = f"Environment validation failed after native completion: {error}" + else: + row["environment_validation_state"] = "passed" if unchanged else "changed" + row["execution_valid"] = native_valid and unchanged + row["within_target"] = row["execution_valid"] and row["elapsed_seconds"] <= target + if not unchanged: + row["error"] = "Source, binary, build or environment changed during native execution" + publish(Path(row["native_record"]).parent / "run.json", row) + case_report["all_measured_runs_within_target"] = row["within_target"] + publish(path, report) + print(f"{case} measurement 1: {row['elapsed_seconds']:.6f}s exit={row['native_exit']} valid={row['execution_valid']}", flush=True) + if not row["execution_valid"]: + return 1 + report["execution_complete"] = True + publish(path, report) + return 0 + + +def link_comparisons(output_dir): + """Attach closed comparator reports by hashes without changing raw timing.""" + path = output_dir / "measurements.json" + report = json.loads(path.read_text(encoding="utf-8")) + for case in report["cases"]: + for row in case["runs"]: + comparison_path = Path(row["native_record"]).parent / "comparison.json" + if not comparison_path.is_file(): + continue + comparison = json.loads(comparison_path.read_text(encoding="utf-8")) + identities = comparison.get("identities", {}) + native_path = Path(row["native_record"]) + native = json.loads(native_path.read_text(encoding="utf-8")) + if (sha256(native_path) != row["native_record_sha256"] + or row["elapsed_seconds"] != native["elapsed_seconds"] + or sha256(Path(row["scratch_input"])) != row["scratch_sha256"] + or sha256(Path(row["result"])) != row["result_sha256"] + or identities.get("input", {}).get("sha256") != row["scratch_sha256"] + or identities.get("results", {}).get("sha256") != row["result_sha256"] + or comparison.get("case") != case["case"]): + raise ValueError("Comparison does not identify the recorded native run/result") + if case["case"].startswith("hexa_") and comparison.get("blocking_passed") and ( + comparison.get("execution", {}).get("record_sha256") != row["native_record_sha256"]): + raise ValueError("Hex8 comparator did not verify this execution record") + row["comparison_status"] = "passed" if ( + row["execution_valid"] and comparison.get("precheck", {}).get("passed") + and comparison.get("blocking_passed") and comparison.get("verdict") == "passed" + ) else "failed" + row["comparison_report"] = str(comparison_path) + row["comparison_report_sha256"] = sha256(comparison_path) + report["comparisons_complete"] = (report["execution_complete"] and all( + row["comparison_status"] != "pending" for case in report["cases"] for row in case["runs"])) + report["all_comparisons_passed"] = (report["comparisons_complete"] and all( + row["comparison_status"] == "passed" for case in report["cases"] for row in case["runs"])) + publish(path, report) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--exe", type=Path) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--link-comparisons", action="store_true", + help="Attach existing per-run comparison.json by native/input/result hashes.") + args = parser.parse_args() + if args.link_comparisons: + try: + return link_comparisons(args.output_dir.resolve()) + except (OSError, ValueError, KeyError) as error: + print(f"comparison link failed: {error}", flush=True) + return 1 + if args.exe is None: + parser.error("--exe is required for measurement") + if not args.exe.is_file(): + parser.error("--exe must be an existing executable") + if args.output_dir.exists() and (not args.output_dir.is_dir() or any(args.output_dir.iterdir())): + parser.error("--output-dir must be new or empty") + try: + return measure(args.exe.resolve(), args.output_dir.resolve()) + except (OSError, ValueError, KeyboardInterrupt) as error: + print(f"measurement failed: {error}", flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/fesa/CMakeLists.txt b/src/fesa/CMakeLists.txt index f254d81..77a683d 100644 --- a/src/fesa/CMakeLists.txt +++ b/src/fesa/CMakeLists.txt @@ -131,6 +131,15 @@ target_compile_options( $<$:/W4 /WX /EHsc> ) +# Profile the optimized production build with the same binary used for timing. +foreach(_fesa_target fesa_solver fesa_cli) + target_compile_options(${_fesa_target} PRIVATE + $<$,$>:/Zi>) +endforeach() +target_link_options(fesa_cli PRIVATE + $<$,$>:/DEBUG /OPT:REF /OPT:ICF>) +unset(_fesa_target) + if(WIN32) if(NOT TARGET MKL::mkl_intel_thread OR NOT TARGET MKL::mkl_core OR NOT OMP_DLL_DIR OR NOT OMP_DLLNAME) diff --git a/tests/test_performance_measure.py b/tests/test_performance_measure.py new file mode 100644 index 0000000..948b5b5 --- /dev/null +++ b/tests/test_performance_measure.py @@ -0,0 +1,268 @@ +"""Exercise process boundaries and preserved raw evidence, not solver numerics.""" +import importlib.util +import json +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("measure", ROOT / "scripts/performance/measure.py") +M = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(M) + + +@pytest.fixture +def source(tmp_path): + path = tmp_path / "model.inp" + path.write_text("from pathlib import Path\nimport time\ntime.sleep(0.08)\n" + "Path(__file__).with_name('model_results.h5').write_bytes(b'fresh')\n") + return path + + +def test_real_spawn_isolated_bytes_and_pending_comparison(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {"frames": 101}) + before = source.read_bytes() + rows = [M.run_once(Path(sys.executable), source, "fixture", tmp_path / str(i), i) + for i in range(2)] + assert source.read_bytes() == before + for i, row in enumerate(rows): + assert row["native_exit"] == 0 and row["execution_state"] == "completed" + assert row["elapsed_seconds"] >= 0.08 + assert row["comparison_status"] == "pending" + assert row["measurement_index"] == i + native = json.loads(Path(row["native_record"]).read_text()) + assert len(native["command"]) == 2 + assert native["original_sha256"] == native["scratch_sha256"] + assert native["scratch_unchanged"] and native["source_unchanged"] + assert rows[0]["scratch_input"] != rows[1]["scratch_input"] + + +def test_timer_excludes_preparation_and_validation(tmp_path, source, monkeypatch): + events = [] + real_copy, real_spawn = M.shutil.copyfile, M.subprocess.Popen + monkeypatch.setattr(M.shutil, "copyfile", lambda *a: (events.append("copy"), real_copy(*a))[1]) + monkeypatch.setattr(M.time, "perf_counter", lambda: (events.append("clock"), float(len(events)))[1]) + monkeypatch.setattr(M.subprocess, "Popen", lambda *a, **kw: (events.append("spawn"), real_spawn(*a, **kw))[1]) + monkeypatch.setattr(M, "validate_result", lambda *a: (events.append("validate"), {})[1]) + M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert events == ["copy", "clock", "spawn", "clock", "validate"] + + +def test_nonzero_with_output_is_not_success(tmp_path, source, monkeypatch): + source.write_text(source.read_text() + "raise SystemExit(7)\n") + monkeypatch.setattr(M, "validate_result", lambda *a: pytest.fail("invalid native exit validated")) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert row["native_exit"] == 7 and not row["execution_valid"] + assert row["result_sha256"] is not None + + +def test_existing_directory_and_old_output_rejected(tmp_path, source): + run = tmp_path / "run" + run.mkdir() + old = run / "model_results.h5" + old.write_bytes(b"old") + with pytest.raises(FileExistsError): + M.run_once(Path(sys.executable), source, "fixture", run, 0) + assert old.read_bytes() == b"old" + + +def test_launch_failure_never_invents_result_hash(tmp_path, source): + row = M.run_once(tmp_path / "absent.exe", source, "fixture", tmp_path / "run", 0) + assert row["execution_state"] == "launch-failed" and row["native_exit"] is None + assert row["result_sha256"] is None and not row["execution_valid"] + + +def test_input_mutation_invalidates_result(tmp_path, source, monkeypatch): + source.write_text(source.read_text() + "Path(__file__).write_text('changed')\n") + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0) + assert not row["execution_valid"] + assert not json.loads(Path(row["native_record"]).read_text())["scratch_unchanged"] + + +def test_interrupted_run_and_prior_case_are_preserved(tmp_path, source, monkeypatch): + real = M.subprocess.Popen + calls = 0 + class Interrupted: + def __init__(self, *a, **kw): + self.proc = real(*a, **kw) + def wait(self, timeout=None): + raise KeyboardInterrupt + def kill(self): + self.proc.kill() + def communicate(self): + return self.proc.communicate() + @property + def returncode(self): + return self.proc.returncode + def spawn(*a, **kw): + nonlocal calls + calls += 1 + return real(*a, **kw) if calls == 1 else Interrupted(*a, **kw) + monkeypatch.setattr(M.subprocess, "Popen", spawn) + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) == 1 + report = json.loads((out / "measurements.json").read_text()) + assert len(report["cases"]) == 2 + assert report["cases"][0]["runs"][0]["execution_state"] == "completed" + assert report["cases"][1]["runs"][0]["execution_state"] == "interrupted" + assert not report["cases"][1]["all_measured_runs_within_target"] + + +def test_timeout_preserves_actual_exit(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: pytest.fail("timed out")) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "run", 0, timeout=0.001) + assert row["execution_state"] == "timed-out" + assert type(row["native_exit"]) is int and not row["execution_valid"] + + +def test_native_zero_missing_or_invalid_result_fails(tmp_path, source, monkeypatch): + source.write_text("pass\n") + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "missing", 0) + assert row["native_exit"] == 0 and not row["execution_valid"] + assert row["result_sha256"] is None + source.write_text("from pathlib import Path\nPath(__file__).with_name('model_results.h5').write_bytes(b'bad schema')\n") + def reject(*a): + raise ValueError("invalid inventory") + monkeypatch.setattr(M, "validate_result", reject) + row = M.run_once(Path(sys.executable), source, "fixture", tmp_path / "invalid", 0) + assert row["native_exit"] == 0 and not row["execution_valid"] + assert row["error"] == "invalid inventory" + + +def test_slow_runs_are_retained_and_exit_zero_is_not_time_pass(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 0.001)]) == 0 + report = json.loads((out / "measurements.json").read_text()) + case = report["cases"][0] + assert report["execution_complete"] and len(case["runs"]) == 1 + assert not case["all_measured_runs_within_target"] + assert case["runs"][0]["kind"] == "measurement" + assert case["runs"][0]["measurement_index"] == 1 + assert all(r["comparison_status"] == "pending" for r in case["runs"]) + + +def test_changed_binary_or_source_blocks_group(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + captures = iter([{"sha": "before"}, {"sha": "before"}, {"sha": "after"}]) + monkeypatch.setattr(M, "collect_environment", lambda *a: next(captures)) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 1)]) == 1 + row = json.loads((out / "measurements.json").read_text())["cases"][0]["runs"][0] + assert row["native_exit"] == 0 and not row["execution_valid"] + + +def test_cli_bad_arguments_are_exit_two(tmp_path): + proc = subprocess.run([sys.executable, str(ROOT / "scripts/performance/measure.py"), + "--exe", str(tmp_path / "missing.exe"), "--output-dir", str(tmp_path)], + capture_output=True) + assert proc.returncode == 2 + + +def test_comparison_link_preserves_raw_time_and_rejects_wrong_result(tmp_path, source, monkeypatch): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + assert M.measure(Path(sys.executable), out, cases=[("fixture", source, 1)]) == 0 + before = json.loads((out / "measurements.json").read_text()) + row = before["cases"][0]["runs"][0] + comparison_path = Path(row["native_record"]).parent / "comparison.json" + comparison = {"case": "fixture", "identities": { + "input": {"sha256": row["scratch_sha256"]}, "results": {"sha256": row["result_sha256"]}}, + "precheck": {"passed": True}, "blocking_passed": True, "verdict": "passed"} + M.publish(comparison_path, comparison) + assert M.link_comparisons(out) == 0 + after = json.loads((out / "measurements.json").read_text()) + assert after["cases"][0]["runs"][0]["comparison_status"] == "passed" + assert [r["elapsed_seconds"] for r in after["cases"][0]["runs"]] == [ + r["elapsed_seconds"] for r in before["cases"][0]["runs"]] + comparison["identities"]["results"]["sha256"] = "other run" + M.publish(comparison_path, comparison) + with pytest.raises(ValueError): + M.link_comparisons(out) + assert json.loads((out / "measurements.json").read_text()) == after + + +def test_result_validation_interrupt_preserves_completed_native_and_prior_case(tmp_path, source, monkeypatch): + calls = 0 + def validate(*a): + nonlocal calls + calls += 1 + if calls == 2: + raise KeyboardInterrupt + return {} + monkeypatch.setattr(M, "validate_result", validate) + monkeypatch.setattr(M, "collect_environment", lambda *a: {"fixed": True}) + out = tmp_path / "artifacts" + try: + code = M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) + except KeyboardInterrupt: + pytest.fail("Result validation interrupt escaped without a completed native row") + assert code == 1 + report = json.loads((out / "measurements.json").read_text()) + assert len(report["cases"]) == 2 and not report["execution_complete"] + assert report["cases"][0]["runs"][0]["execution_valid"] + row = report["cases"][1]["runs"][0] + native = json.loads(Path(row["native_record"]).read_text()) + assert row["native_exit"] == native["native_exit"] == 0 + assert row["execution_state"] == native["execution_state"] == "completed" + assert row["elapsed_seconds"] == native["elapsed_seconds"] + assert row["result_sha256"] == native["result_sha256"] is not None + assert row["result_validation_state"] == "interrupted" and not row["execution_valid"] + assert row["comparison_status"] == "pending" + + +@pytest.mark.parametrize("exception,state", [(KeyboardInterrupt, "interrupted"), (OSError, "failed")]) +def test_post_native_environment_failure_preserves_current_and_prior_case(tmp_path, source, monkeypatch, exception, state): + monkeypatch.setattr(M, "validate_result", lambda *a: {}) + out = tmp_path / "artifacts" + def collect(*a): + if (out / "second/run-1/native.json").is_file(): + raise exception("post-native environment unavailable") + return {"fixed": True} + monkeypatch.setattr(M, "collect_environment", collect) + try: + code = M.measure(Path(sys.executable), out, cases=[("first", source, 1), ("second", source, 1)]) + except (KeyboardInterrupt, OSError): + pytest.fail("Post-native environment failure escaped without preserving the current row") + assert code == 1 + report = json.loads((out / "measurements.json").read_text()) + assert report["cases"][0]["runs"][0]["execution_valid"] + row = report["cases"][1]["runs"][0] + native = json.loads(Path(row["native_record"]).read_text()) + assert row["native_exit"] == native["native_exit"] == 0 + assert row["execution_state"] == native["execution_state"] == "completed" + assert row["elapsed_seconds"] == native["elapsed_seconds"] + assert row["result_sha256"] == native["result_sha256"] is not None + assert row["environment_validation_state"] == state + assert not row["execution_valid"] and row["error"] + assert not row["within_target"] and not report["execution_complete"] + assert json.loads((out / "second/run-1/run.json").read_text()) == row + + +@pytest.mark.parametrize("case,value,code", [ + ("hexa_nonlinearstatic", float("nan"), "nonfinite-result-value"), + ("hexa_nonlinearstatic", float("inf"), "nonfinite-result-value"), + ("hexa_dynnonlinear", 1e-12, "nonzero-initial-result-state"), +]) +def test_hex8_precheck_rejects_invalid_physical_values(tmp_path, case, value, code): + import h5py + helper_spec = importlib.util.spec_from_file_location("hex8_fixture", ROOT / "tests/test_compare_hex8.py") + helper = importlib.util.module_from_spec(helper_spec) + helper_spec.loader.exec_module(helper) + scratch = tmp_path / f"{case}.inp" + M.shutil.copyfile(ROOT / "reference/hexa" / scratch.name, scratch) + result = tmp_path / f"{case}_results.h5" + model = helper.COMPARE.input_inventory(scratch, case) + helper.make_result(result, model) + with h5py.File(result, "r+") as file: + file["steps/Step-1/frames/0/nodal/displacement"][0, 0] = value + with pytest.raises(Exception) as error: + M.validate_result(case, scratch, result) + assert error.value.code == code