From 2641d2787586c3fdfe769415fb9f129c7d3d2751 Mon Sep 17 00:00:00 2001 From: "KOKO\\Mimi" Date: Sun, 6 Sep 2026 12:43:02 +0900 Subject: [PATCH] chore(mitc4-shell-feast-eigen-execution-prep): WIP classifier correction with full verification blocked --- scripts/execute.py | 41 +++++++++++++- tests/test_execute.py | 124 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/scripts/execute.py b/scripts/execute.py index 5927894..a29d70c 100644 --- a/scripts/execute.py +++ b/scripts/execute.py @@ -1005,12 +1005,51 @@ class StepExecutor: return output + @staticmethod + def _stdout_has_helper_failure(stdout: str) -> bool: + """JSONL source/tool payloads are not runtime diagnostics.""" + marker = "orchestrator_helper_launch_failed" + for line in stdout.splitlines(): + if marker not in line.lower(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + pass + else: + if not isinstance(event, dict): + continue + if event.get("type") in {"error", "turn.failed"}: + error = event.get("message", "") if event["type"] == "error" else event.get("error", "") + if marker in json.dumps(error).lower(): + return True + continue + item = event.get("item") + if ( + not isinstance(item, dict) + or item.get("type") != "command_execution" + or item.get("exit_code") in (None, 0) + ): + continue + line = item.get("aggregated_output", "") + # Plain output must have a diagnostic prefix, including inside a failed command. + if re.search( + r"(?im)^\s*(?:(?:windows sandbox:\s*)?orchestrator_helper_launch_failed\b|" + r"\S+\s+ERROR\b[^\r\n]*\borchestrator_helper_launch_failed\b)", + line, + ): + return True + return False + @staticmethod def _codex_environment_failure(output: dict) -> Optional[str]: diagnostic = ( f"{output.get('stderr', '')}\n{output.get('stdout', '')}" ).lower() - if "orchestrator_helper_launch_failed" in diagnostic: + if ( + "orchestrator_helper_launch_failed" in output.get("stderr", "").lower() + or StepExecutor._stdout_has_helper_failure(output.get("stdout", "")) + ): return "Codex Windows sandbox helper를 실행할 수 없습니다." if output.get("exitCode", 0) == 0: return None diff --git a/tests/test_execute.py b/tests/test_execute.py index 38ae64e..6cbca76 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -323,6 +323,71 @@ def test_codex_environment_failure_detects_missing_sandbox_helper_on_zero_exit() ) +def source_echo_event(text): + return json.dumps({ + "type": "item.completed", + "item": { + "type": "command_execution", "command": "Get-Content scripts/execute.py", + "aggregated_output": text, "exit_code": 0, "status": "completed", + }, + }) + + +@pytest.mark.parametrize("stdout", [ + 'Source excerpt: if "orchestrator_helper_launch_failed" in diagnostic:', + source_echo_event('if "orchestrator_helper_launch_failed" in diagnostic:'), + source_echo_event("windows sandbox: orchestrator_helper_launch_failed: helper not found"), + source_echo_event(json.dumps({"type": "error", "message": "orchestrator_helper_launch_failed"})), + json.dumps({"type": "item.completed", "item": { + "type": "command_execution", "exit_code": 0, + "command": "rg orchestrator_helper_launch_failed scripts/execute.py", + "aggregated_output": "", "status": "completed", + }}), + json.dumps({"type": "item.completed", "item": { + "type": "agent_message", "text": "Source contains orchestrator_helper_launch_failed", + }}), +], ids=["plain-excerpt", "source-code", "quoted-runtime-line", "quoted-runtime-event", + "command-argument", "agent-source-description"]) +def test_environment_classifier_ignores_successful_source_echo(stdout): + output = {"exitCode": 0, "stdout": stdout, "stderr": ""} + + assert execute.StepExecutor._codex_environment_failure(output) is None + + +@pytest.mark.parametrize("process_exit", [0, 1]) +@pytest.mark.parametrize("route", [ + "stderr", "stdout", "timestamped-stdout", "error-event", "turn-failed", + "failed-command", "mixed-stderr", "mixed-stdout", +]) +def test_environment_classifier_retains_runtime_helper_failure(process_exit, route): + diagnostic = "windows sandbox: orchestrator_helper_launch_failed: helper not found" + output = {"exitCode": process_exit, "stdout": "", "stderr": ""} + if route in {"stderr", "mixed-stderr"}: + output["stderr"] = diagnostic + elif route == "error-event": + output["stdout"] = json.dumps({"type": "error", "message": diagnostic}) + elif route == "turn-failed": + output["stdout"] = json.dumps({"type": "turn.failed", "error": {"message": diagnostic}}) + elif route == "failed-command": + output["stdout"] = json.dumps({"type": "item.completed", "item": { + "type": "command_execution", "status": "failed", "exit_code": 1, + "aggregated_output": diagnostic, + }}) + else: + output["stdout"] = diagnostic + if route == "timestamped-stdout": + output["stdout"] = "2026-09-06T01:09:28Z ERROR codex_core::tools: " + diagnostic + if route.startswith("mixed-"): + output["stdout"] = source_echo_event(diagnostic) + "\n" + output["stdout"] + + reason = "Codex Windows sandbox helper를 실행할 수 없습니다." + assert execute.StepExecutor._codex_environment_failure(output) == reason + executor = object.__new__(execute.StepExecutor) + assert executor._classify_attempt({"status": "pending"}, output, None) == ( + "environment", "blocked", reason, + ) + + def test_run_git_decodes_output_as_utf8(tmp_path, monkeypatch): invocation = {} @@ -438,6 +503,65 @@ def run_step(runtime): return runtime.executor._execute_single_step(index["steps"][0], "") +@pytest.mark.parametrize("status,process_exit,timeouts", [ + ("completed", 0, 0), ("completed", 1, 0), + ("pending", 0, 0), ("pending", 1, 0), ("pending", 124, 0), + ("error", 0, 0), ("error", 1, 0), ("error", 124, 0), + ("blocked", 0, 0), ("blocked", 1, 0), ("blocked", 124, 0), + ("completed", 0, 1), ("pending", 0, 2), + ("error", 124, 1), ("blocked", 124, 1), +]) +def test_successful_source_echo_preserves_terminal_and_retry( + step_runtime, status, process_exit, timeouts +): + runtime = step_runtime + echo = source_echo_event("windows sandbox: orchestrator_helper_launch_failed: helper not found") + response = {"status": status, "exit": process_exit, "reason": "original result", "stdout": echo} + timeout = subprocess.TimeoutExpired("codex", 1800, output=echo) + terminal_timeout = timeouts and status in {"error", "blocked"} + + def report_then_timeout(runtime): + index = runtime.executor._read_phase_index() + current = index["steps"][0] + current["status"] = status + current["error_message" if status == "error" else "blocked_reason"] = "original result" + runtime.executor._write_json(runtime.index_file, index) + return timeout + + runtime.responses = [report_then_timeout] if terminal_timeout else [timeout] * timeouts + [response] + if status == "completed" and process_exit == 0: + assert run_step(runtime) + expected = "completed" + else: + with pytest.raises(SystemExit) as stopped: + run_step(runtime) + assert stopped.value.code == (2 if status == "blocked" else 1) + expected = "blocked" if status == "blocked" else "error" + current = runtime.executor._read_phase_index()["steps"][0] + history = current["attempt_history"] + calls = 1 if terminal_timeout else min(timeouts + 1, 2) + assert len(runtime.calls) == len(history) == calls + assert [record["attempt"] for record in history] == list(range(1, calls + 1)) + assert current["status"] == expected + classification = ( + "transport-timeout" if timeouts == 2 else + "reported-" + status if status in {"error", "blocked"} else + "completed" if expected == "completed" else + "inconsistent-result" if status == "completed" else + "process-error" if process_exit else "missing-result" + ) + assert history[-1]["classification"] == classification + assert history[-1]["exit_code"] == (124 if timeouts == 2 or terminal_timeout else process_exit) + if timeouts and not terminal_timeout: + assert history[0]["classification"] == "transport-timeout" + assert "Codex timed out after 1800 seconds." in history[0]["diagnostic"] + if status in {"error", "blocked"}: + field = "error_message" if status == "error" else "blocked_reason" + assert current[field] == history[-1]["reason"] == history[-1]["reported_reason"] == "original result" + if not timeouts: + assert history[-1]["diagnostic"] == echo + + @pytest.mark.parametrize("status,expected_exit", [("error", 1), ("blocked", 2)]) @pytest.mark.parametrize("process_exit", [0, 1, 124]) def test_explicit_failure_preserves_reason_without_retry(step_runtime, status, expected_exit, process_exit):