import json from pathlib import Path import re import tomllib import pytest from test_execute import step_runtime ROOT = Path(__file__).resolve().parents[1] AGENT_DIR = ROOT / ".codex" / "agents" SKILL_DIR = ROOT / ".codex" / "skills" EXPECTED_AGENTS = { "coordinator-agent", "correction-agent", "formulation-agent", "implementation-agent", "implementation-planning-agent", "io-definition-agent", "numerical-review-agent", "physics-evaluation-agent", "release-agent", "requirement-agent", "research-agent", } EXPECTED_SKILLS = { "fem-theory-query", "fem-implementation-tdd", "fem-formulation-spec", "fem-io-contract", "fem-numerical-review", "fem-physics-review", "fem-release-readiness", "fem-requirements-baseline", "fem-research-evidence", } LEGACY_OUTPUT_DIRS = { "coordination", "requirements", "research", "formulations", "numerical-reviews", "io-definitions", "reference-models", "implementation-plans", "build-test-reports", "corrections", "reference-verifications", "physics-evaluations", "releases", } FEATURE_FILES = { "linear-static-3d-euler-beam": { "coordination.md", "requirements.md", "research.md", "formulation.md", "numerical-review.md", "reference-model.md", "io.md", "implementation-plan.md", "implementation-report.md", "build-test.md", "reference-comparison.md", "physics-evaluation.md", "release.md", }, "linear-static-mitc4-shell": { "coordination.md", "requirements.md", "research.md", "formulation.md", "numerical-review.md", "reference-model.md", "io.md", "implementation-plan.md", "build-test.md", "reference-comparison.md", "physics-evaluation.md", "release.md", }, } LIVE_CONTRACT_FILES = ( ROOT / "AGENTS.md", ROOT / "docs" / "SOLVER_AGENT_DESIGN.md", ROOT / "docs" / "SOLVER_SKILL_DESIGN.md", ) AGENT_STAGE_GUIDANCE_FILES = ( ROOT / "AGENTS.md", ROOT / "docs" / "SOLVER_AGENT_DESIGN.md", ) EXPECTED_AGENT_STAGE_MAPPING = ( ("requirements", "requirement-agent", ("requirements.md",)), ("research", "research-agent", ("research.md",)), ("formulation", "formulation-agent", ("formulation.md",)), ( "numerical-review", "numerical-review-agent", ("numerical-review.md", "reference-model.md"), ), ("io-definition", "io-definition-agent", ("io.md",)), ( "implementation-planning", "implementation-planning-agent", ("implementation-plan.md",), ), ( "implementation", "implementation-agent", ("implementation-report.md", "build-test.md", "reference-comparison.md"), ), ("physics-evaluation", "physics-evaluation-agent", ("physics-evaluation.md",)), ("release-readiness", "release-agent", ("release.md",)), ) CANONICAL_FEATURE_DIRS = ( ROOT / "docs" / "linear-static-3d-euler-beam", ROOT / "docs" / "linear-static-mitc4-shell", ) # Plans and specs intentionally retain deleted source paths as migration history. HISTORICAL_PATH_EXEMPT_DIRS = ( ROOT / "docs" / "superpowers" / "plans", ROOT / "docs" / "superpowers" / "specs", ) LEGACY_README_PATTERN = re.compile( r"docs/(?:coordination|requirements|research|formulations|numerical-reviews|" r"io-definitions|reference-models|implementation-plans|build-test-reports|" r"corrections|reference-verifications|physics-evaluations|releases)/README\.md", re.IGNORECASE, ) EXPECTED_SKILL_FRONTMATTER_KEYS = ("name", "description") EXPECTED_SKILL_INTERFACE_KEYS = ( "display_name", "short_description", "default_prompt", ) RETIRED_TOKENS = ( "reference-model-agent", "reference model agent", "build-test-executor-agent", "build/test executor agent", "reference-verification-agent", "reference verification agent", "fesa-reference-models", "fesa-reference-comparison", "docs/coordination/", "docs/requirements/", "docs/research/", "docs/formulations/", "docs/numerical-reviews/", "docs/io-definitions/", "docs/reference-models/", "docs/implementation-plans/", "docs/build-test-reports/", "docs/corrections/", "docs/reference-verifications/", "docs/physics-evaluations/", "docs/releases/", ) def read(path: Path) -> str: return path.read_text(encoding="utf-8") def live_contract_text() -> str: paths = list(LIVE_CONTRACT_FILES) paths.extend(sorted(AGENT_DIR.glob("*.toml"))) paths.extend(sorted(SKILL_DIR.glob("*/SKILL.md"))) return "\n".join(read(path) for path in paths) def markdown_stage_mapping(path: Path) -> tuple[tuple[str, str, tuple[str, ...]], ...]: heading = ( "## Agent/Skill Workflow" if path.name == "AGENTS.md" else "## 4. 단계와 소유권" ) section = read(path).split(heading, maxsplit=1)[1].split("\n## ", maxsplit=1)[0] rows = [] for line in section.splitlines(): if not line.lstrip().startswith("|"): continue columns = [column.strip().replace("`", "") for column in line.strip().split("|")[1:-1]] if len(columns) < 3: continue stage = re.sub(r"^\d+\.\s+", "", columns[0]) if re.fullmatch(r"[a-z]+(?:-[a-z]+)*", stage): reports = tuple(re.findall(r"\b[\w-]+\.md\b", columns[-1])) rows.append((stage, columns[1], reports)) return tuple(rows) def operational_markdown_paths() -> tuple[Path, ...]: paths = [ path for feature_dir in CANONICAL_FEATURE_DIRS for path in sorted(feature_dir.glob("*.md")) ] paths.extend(sorted(ROOT.glob("phases/**/step*.md"))) return tuple(paths) def parse_skill_frontmatter(path: Path) -> dict[str, str]: lines = read(path).splitlines() assert lines and lines[0] == "---", path assert "---" in lines[1:], path closing_index = lines.index("---", 1) field_lines = lines[1:closing_index] assert len(field_lines) == len(EXPECTED_SKILL_FRONTMATTER_KEYS), path parsed = {} for line in field_lines: match = re.fullmatch(r"([a-z_]+): ([^\s].*)", line) assert match is not None, (path, line) key, value = match.groups() assert key not in parsed, (path, key) parsed[key] = value assert tuple(parsed) == EXPECTED_SKILL_FRONTMATTER_KEYS, path assert re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", parsed["name"]), path assert parsed["description"].startswith("Use when "), path return parsed def parse_skill_interface(path: Path) -> dict[str, str]: lines = [line for line in read(path).splitlines() if line] assert lines and lines[0] == "interface:", path assert len(lines) == len(EXPECTED_SKILL_INTERFACE_KEYS) + 1, path parsed = {} for line in lines[1:]: match = re.fullmatch(r' ([a-z_]+): ("(?:[^"\\]|\\.)*")', line) assert match is not None, (path, line) key, quoted_value = match.groups() assert key not in parsed, (path, key) parsed[key] = json.loads(quoted_value) assert tuple(parsed) == EXPECTED_SKILL_INTERFACE_KEYS, path assert all(parsed.values()), path return parsed def test_agent_inventory_is_consolidated(): actual = {path.stem for path in AGENT_DIR.glob("*.toml")} assert actual == EXPECTED_AGENTS def test_project_skill_inventory_is_consolidated(): actual = { path.name for path in SKILL_DIR.iterdir() if path.is_dir() and (path / "SKILL.md").is_file() } assert actual == EXPECTED_SKILLS def test_existing_feature_artifacts_are_bundled(): docs_dir = ROOT / "docs" for legacy_dir in LEGACY_OUTPUT_DIRS: assert not (docs_dir / legacy_dir).exists(), legacy_dir for feature_id, expected_files in FEATURE_FILES.items(): feature_dir = docs_dir / feature_id actual_files = {path.name for path in feature_dir.glob("*.md")} # Preserve historical artifacts while allowing Feature management and linked details. assert expected_files <= actual_files def test_numerical_review_absorbs_reference_model_contract(): assert not (AGENT_DIR / "reference-model-agent.toml").exists() assert not (SKILL_DIR / "fesa-reference-models").exists() text = read(SKILL_DIR / "fem-numerical-review" / "SKILL.md") outputs = set(re.findall(r"docs//([\w-]+\.md)", text)) assert {"numerical-review.md", "reference-model.md"} <= outputs def test_implementation_absorbs_build_test_and_reference_comparison(): assert not (AGENT_DIR / "build-test-executor-agent.toml").exists() assert not (AGENT_DIR / "reference-verification-agent.toml").exists() assert not (SKILL_DIR / "fesa-reference-comparison").exists() text = read(SKILL_DIR / "fem-implementation-tdd" / "SKILL.md") outputs = set(re.findall(r"docs//([\w-]+\.md)", text)) assert { "implementation-plan.md", "implementation-report.md", "build-test.md", "reference-comparison.md", "corrections.md", } <= outputs def test_implementation_verification_reference_is_resolvable(): skill = SKILL_DIR / "fem-implementation-tdd" / "SKILL.md" links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", read(skill)) assert "references/toolchain-and-verification.md" in links for link in links: assert (skill.parent / link).is_file(), link def test_p_agent_001_implementation_guidance_resolves_coding_style_contract(): parsed = tomllib.loads(read(AGENT_DIR / "implementation-agent.toml")) instructions = parsed["developer_instructions"] assert "AGENTS.md" in instructions project_links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", read(ROOT / "AGENTS.md")) assert "docs/CODINGSTYLE.md" in project_links assert (ROOT / "docs/CODINGSTYLE.md").is_file() def test_agent_profiles_reference_shared_authority_and_feature_context(): for path in sorted(AGENT_DIR.glob("*.toml")): instructions = tomllib.loads(read(path))["developer_instructions"] for document in ( "AGENTS.md", "docs/SOLVER_AGENT_DESIGN.md", "docs/FEATURE_DOCUMENTATION.md", "FEATURED_AGENTS.md", "PRD.md", "ADR.md", ): assert document in instructions, (path.name, document) for skill_name in re.findall(r"\$([a-z0-9-]+)", instructions): candidates = ( SKILL_DIR / skill_name / "SKILL.md", ROOT / ".agents/skills" / skill_name / "SKILL.md", ) assert any(candidate.is_file() for candidate in candidates), (path, skill_name) def test_live_contracts_have_no_retired_names_or_paths(): text = live_contract_text().lower() assert "docs//" in text for token in RETIRED_TOKENS: assert token not in text, token def test_live_guidance_declares_consistent_ordered_nine_stage_workflow(): for path in LIVE_CONTRACT_FILES: text = read(path) assert "docs//" in text, path for path in AGENT_STAGE_GUIDANCE_FILES: assert markdown_stage_mapping(path) == EXPECTED_AGENT_STAGE_MAPPING, path def test_operational_documents_do_not_use_deleted_stage_readmes(): paths = operational_markdown_paths() assert paths for path in paths: assert not any(path.is_relative_to(directory) for directory in HISTORICAL_PATH_EXEMPT_DIRS) match = LEGACY_README_PATTERN.search(read(path)) assert match is None, (path, match.group(0) if match else None) def test_mitc4_live_authority_links_use_feature_bundle(): text = read(ROOT / "docs" / "MITC4_SUPP.md") authority_section = text.split("라인 참조", maxsplit=1)[0] for path in ( "docs/linear-static-mitc4-shell/requirements.md", "docs/linear-static-mitc4-shell/formulation.md", "docs/linear-static-mitc4-shell/io.md", "docs/linear-static-mitc4-shell/reference-model.md", "docs/linear-static-mitc4-shell/numerical-review.md", "docs/linear-static-mitc4-shell/release.md", ): assert path in authority_section for legacy_path in ( "docs/requirements/linear-static-mitc4-shell.md", "docs/formulations/mitc4-shell-formulation.md", "docs/io-definitions/linear-static-mitc4-shell-io.md", "docs/reference-models/linear-static-mitc4-shell-reference-models.md", "docs/numerical-reviews/linear-static-mitc4-shell-review.md", "docs/releases/linear-static-mitc4-shell-release.md", ): assert legacy_path not in authority_section def test_agent_toml_and_limited_skill_yaml_schemas_are_parseable(): for path in sorted(AGENT_DIR.glob("*.toml")): parsed = tomllib.loads(read(path)) assert parsed["name"] == path.stem assert parsed["model_reasoning_effort"] == "xhigh" assert parsed["sandbox_mode"] == "workspace-write" assert parsed["developer_instructions"].strip(), path for skill_name in EXPECTED_SKILLS: skill_dir = SKILL_DIR / skill_name frontmatter = parse_skill_frontmatter(skill_dir / "SKILL.md") interface = parse_skill_interface(skill_dir / "agents" / "openai.yaml") assert frontmatter["name"] == skill_name assert f"${skill_name}" in interface["default_prompt"] @pytest.mark.parametrize("document", [".agents/skills/harness/SKILL.md", "docs/HARNESS_WORKFLOW.md"]) def test_documented_phase_example_passes_executor_preflight(step_runtime, document): """Exercise the documented schema with its real consumer, not prose matching.""" from scripts import execute examples = [json.loads(block) for block in re.findall(r"```json\s*\n(.*?)\n```", read(ROOT / document), re.DOTALL)] phase = next(example for example in examples if "steps" in example) phase["project"] = "Harness Tests" phase["phase"] = "runtime" phase["steps"][0]["name"] = "runtime" runtime = step_runtime runtime.executor._write_json(runtime.index_file, phase) executor = execute.StepExecutor("runtime") executor._preflight_repository() assert runtime.calls == []