fix(harness): retry timed-out Codex steps
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import scripts.execute as execute_module
|
||||
from scripts.execute import (
|
||||
CodexEnvironmentError,
|
||||
StepExecutor,
|
||||
build_codex_command,
|
||||
build_codex_environment,
|
||||
configure_standard_streams,
|
||||
@@ -105,6 +108,95 @@ def test_run_utf8_process_sends_prompt_as_utf8(tmp_path):
|
||||
assert result.stdout == prompt
|
||||
|
||||
|
||||
def test_invoke_codex_records_timeout_with_partial_output(tmp_path, monkeypatch):
|
||||
phase_dir = tmp_path / "phases" / "test-phase"
|
||||
phase_dir.mkdir(parents=True)
|
||||
(phase_dir / "step0.md").write_text("# Timed step\n", encoding="utf-8")
|
||||
codex = tmp_path / "codex.exe"
|
||||
codex.touch()
|
||||
|
||||
executor = StepExecutor.__new__(StepExecutor)
|
||||
executor._root = str(tmp_path)
|
||||
executor._phase_dir = phase_dir
|
||||
executor._codex_add_dirs = ()
|
||||
|
||||
monkeypatch.setattr(
|
||||
execute_module.shutil, "which", lambda executable: str(codex)
|
||||
)
|
||||
|
||||
def raise_timeout(*args, **kwargs):
|
||||
raise subprocess.TimeoutExpired(
|
||||
cmd=["codex", "exec"],
|
||||
timeout=1800,
|
||||
output=b"partial stdout",
|
||||
stderr=b"partial stderr",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(execute_module, "run_utf8_process", raise_timeout)
|
||||
|
||||
output = executor._invoke_codex(
|
||||
{"step": 0, "name": "timed-step"}, "preamble\n"
|
||||
)
|
||||
|
||||
assert output["exitCode"] == 124
|
||||
assert output["stdout"] == "partial stdout"
|
||||
assert "partial stderr" in output["stderr"]
|
||||
assert "timed out after 1800 seconds" in output["stderr"]
|
||||
assert (
|
||||
execute_module.json.loads(
|
||||
(phase_dir / "step0-output.json").read_text(encoding="utf-8")
|
||||
)
|
||||
== output
|
||||
)
|
||||
|
||||
|
||||
def test_execute_single_step_retries_timeout_three_times(tmp_path, monkeypatch):
|
||||
index_file = tmp_path / "index.json"
|
||||
index_file.write_text(
|
||||
execute_module.json.dumps(
|
||||
{
|
||||
"steps": [
|
||||
{"step": 0, "name": "timed-step", "status": "pending"}
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
executor = StepExecutor.__new__(StepExecutor)
|
||||
executor._index_file = index_file
|
||||
executor._total = 1
|
||||
executor._project = "FESA"
|
||||
executor._phase_dir_name = "test-phase"
|
||||
attempts = 0
|
||||
|
||||
def timeout_output(step, preamble):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
return {
|
||||
"step": 0,
|
||||
"name": "timed-step",
|
||||
"exitCode": 124,
|
||||
"stdout": "",
|
||||
"stderr": "Codex timed out after 1800 seconds.",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(executor, "_invoke_codex", timeout_output)
|
||||
monkeypatch.setattr(executor, "_commit_step", lambda *args: None)
|
||||
monkeypatch.setattr(executor, "_update_top_index", lambda *args: None)
|
||||
|
||||
with pytest.raises(SystemExit) as exit_info:
|
||||
executor._execute_single_step(
|
||||
{"step": 0, "name": "timed-step", "status": "pending"}, ""
|
||||
)
|
||||
|
||||
assert exit_info.value.code == 1
|
||||
assert attempts == 3
|
||||
index = execute_module.json.loads(index_file.read_text(encoding="utf-8"))
|
||||
assert index["steps"][0]["status"] == "error"
|
||||
assert "timed out after 1800 seconds" in index["steps"][0]["error_message"]
|
||||
|
||||
|
||||
def test_configure_standard_streams_replaces_unencodable_status_characters():
|
||||
class RecordingStream:
|
||||
def __init__(self):
|
||||
|
||||
Reference in New Issue
Block a user