fix(harness): retry timed-out Codex steps
This commit is contained in:
@@ -343,6 +343,23 @@ class StepExecutor:
|
|||||||
env=environment,
|
env=environment,
|
||||||
timeout=1800,
|
timeout=1800,
|
||||||
)
|
)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
stdout = exc.stdout or ""
|
||||||
|
stderr = exc.stderr or ""
|
||||||
|
if isinstance(stdout, bytes):
|
||||||
|
stdout = stdout.decode("utf-8", errors="replace")
|
||||||
|
if isinstance(stderr, bytes):
|
||||||
|
stderr = stderr.decode("utf-8", errors="replace")
|
||||||
|
timeout_message = (
|
||||||
|
f"Codex timed out after {exc.timeout} seconds."
|
||||||
|
)
|
||||||
|
stderr = f"{stderr}\n{timeout_message}".strip()
|
||||||
|
result = subprocess.CompletedProcess(
|
||||||
|
command,
|
||||||
|
returncode=124,
|
||||||
|
stdout=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
except FileNotFoundError as exc:
|
except FileNotFoundError as exc:
|
||||||
raise CodexEnvironmentError(
|
raise CodexEnvironmentError(
|
||||||
"Codex CLI를 찾을 수 없습니다. codex를 설치하고 PATH를 확인하세요."
|
"Codex CLI를 찾을 수 없습니다. codex를 설치하고 PATH를 확인하세요."
|
||||||
|
|||||||
@@ -1,11 +1,14 @@
|
|||||||
import os
|
import os
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import scripts.execute as execute_module
|
||||||
from scripts.execute import (
|
from scripts.execute import (
|
||||||
CodexEnvironmentError,
|
CodexEnvironmentError,
|
||||||
|
StepExecutor,
|
||||||
build_codex_command,
|
build_codex_command,
|
||||||
build_codex_environment,
|
build_codex_environment,
|
||||||
configure_standard_streams,
|
configure_standard_streams,
|
||||||
@@ -105,6 +108,95 @@ def test_run_utf8_process_sends_prompt_as_utf8(tmp_path):
|
|||||||
assert result.stdout == prompt
|
assert result.stdout == prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_invoke_codex_records_timeout_with_partial_output(tmp_path, monkeypatch):
|
||||||
|
phase_dir = tmp_path / "phases" / "test-phase"
|
||||||
|
phase_dir.mkdir(parents=True)
|
||||||
|
(phase_dir / "step0.md").write_text("# Timed step\n", encoding="utf-8")
|
||||||
|
codex = tmp_path / "codex.exe"
|
||||||
|
codex.touch()
|
||||||
|
|
||||||
|
executor = StepExecutor.__new__(StepExecutor)
|
||||||
|
executor._root = str(tmp_path)
|
||||||
|
executor._phase_dir = phase_dir
|
||||||
|
executor._codex_add_dirs = ()
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
execute_module.shutil, "which", lambda executable: str(codex)
|
||||||
|
)
|
||||||
|
|
||||||
|
def raise_timeout(*args, **kwargs):
|
||||||
|
raise subprocess.TimeoutExpired(
|
||||||
|
cmd=["codex", "exec"],
|
||||||
|
timeout=1800,
|
||||||
|
output=b"partial stdout",
|
||||||
|
stderr=b"partial stderr",
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(execute_module, "run_utf8_process", raise_timeout)
|
||||||
|
|
||||||
|
output = executor._invoke_codex(
|
||||||
|
{"step": 0, "name": "timed-step"}, "preamble\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert output["exitCode"] == 124
|
||||||
|
assert output["stdout"] == "partial stdout"
|
||||||
|
assert "partial stderr" in output["stderr"]
|
||||||
|
assert "timed out after 1800 seconds" in output["stderr"]
|
||||||
|
assert (
|
||||||
|
execute_module.json.loads(
|
||||||
|
(phase_dir / "step0-output.json").read_text(encoding="utf-8")
|
||||||
|
)
|
||||||
|
== output
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_execute_single_step_retries_timeout_three_times(tmp_path, monkeypatch):
|
||||||
|
index_file = tmp_path / "index.json"
|
||||||
|
index_file.write_text(
|
||||||
|
execute_module.json.dumps(
|
||||||
|
{
|
||||||
|
"steps": [
|
||||||
|
{"step": 0, "name": "timed-step", "status": "pending"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
executor = StepExecutor.__new__(StepExecutor)
|
||||||
|
executor._index_file = index_file
|
||||||
|
executor._total = 1
|
||||||
|
executor._project = "FESA"
|
||||||
|
executor._phase_dir_name = "test-phase"
|
||||||
|
attempts = 0
|
||||||
|
|
||||||
|
def timeout_output(step, preamble):
|
||||||
|
nonlocal attempts
|
||||||
|
attempts += 1
|
||||||
|
return {
|
||||||
|
"step": 0,
|
||||||
|
"name": "timed-step",
|
||||||
|
"exitCode": 124,
|
||||||
|
"stdout": "",
|
||||||
|
"stderr": "Codex timed out after 1800 seconds.",
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(executor, "_invoke_codex", timeout_output)
|
||||||
|
monkeypatch.setattr(executor, "_commit_step", lambda *args: None)
|
||||||
|
monkeypatch.setattr(executor, "_update_top_index", lambda *args: None)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit) as exit_info:
|
||||||
|
executor._execute_single_step(
|
||||||
|
{"step": 0, "name": "timed-step", "status": "pending"}, ""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exit_info.value.code == 1
|
||||||
|
assert attempts == 3
|
||||||
|
index = execute_module.json.loads(index_file.read_text(encoding="utf-8"))
|
||||||
|
assert index["steps"][0]["status"] == "error"
|
||||||
|
assert "timed out after 1800 seconds" in index["steps"][0]["error_message"]
|
||||||
|
|
||||||
|
|
||||||
def test_configure_standard_streams_replaces_unencodable_status_characters():
|
def test_configure_standard_streams_replaces_unencodable_status_characters():
|
||||||
class RecordingStream:
|
class RecordingStream:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user