KOKO\Mimi
423 lines
14 KiB
Python
423 lines
14 KiB
Python
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tomllib
|
|
|
|
import pytest
|
|
|
|
from test_execute import step_runtime
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
AGENT_DIR = ROOT / ".codex" / "agents"
|
|
SKILL_DIR = ROOT / ".codex" / "skills"
|
|
|
|
EXPECTED_AGENTS = {
|
|
"coordinator-agent",
|
|
"correction-agent",
|
|
"formulation-agent",
|
|
"implementation-agent",
|
|
"implementation-planning-agent",
|
|
"io-definition-agent",
|
|
"numerical-review-agent",
|
|
"physics-evaluation-agent",
|
|
"release-agent",
|
|
"requirement-agent",
|
|
"research-agent",
|
|
}
|
|
|
|
EXPECTED_SKILLS = {
|
|
"fem-theory-query",
|
|
"fem-implementation-tdd",
|
|
"fem-formulation-spec",
|
|
"fem-io-contract",
|
|
"fem-numerical-review",
|
|
"fem-physics-review",
|
|
"fem-release-readiness",
|
|
"fem-requirements-baseline",
|
|
"fem-research-evidence",
|
|
}
|
|
|
|
LEGACY_OUTPUT_DIRS = {
|
|
"coordination",
|
|
"requirements",
|
|
"research",
|
|
"formulations",
|
|
"numerical-reviews",
|
|
"io-definitions",
|
|
"reference-models",
|
|
"implementation-plans",
|
|
"build-test-reports",
|
|
"corrections",
|
|
"reference-verifications",
|
|
"physics-evaluations",
|
|
"releases",
|
|
}
|
|
|
|
FEATURE_FILES = {
|
|
"linear-static-3d-euler-beam": {
|
|
"coordination.md",
|
|
"requirements.md",
|
|
"research.md",
|
|
"formulation.md",
|
|
"numerical-review.md",
|
|
"reference-model.md",
|
|
"io.md",
|
|
"implementation-plan.md",
|
|
"implementation-report.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"physics-evaluation.md",
|
|
"release.md",
|
|
},
|
|
"linear-static-mitc4-shell": {
|
|
"coordination.md",
|
|
"requirements.md",
|
|
"research.md",
|
|
"formulation.md",
|
|
"numerical-review.md",
|
|
"reference-model.md",
|
|
"io.md",
|
|
"implementation-plan.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"physics-evaluation.md",
|
|
"release.md",
|
|
},
|
|
}
|
|
|
|
LIVE_CONTRACT_FILES = (
|
|
ROOT / "AGENTS.md",
|
|
ROOT / "docs" / "SOLVER_AGENT_DESIGN.md",
|
|
ROOT / "docs" / "SOLVER_SKILL_DESIGN.md",
|
|
)
|
|
|
|
AGENT_STAGE_GUIDANCE_FILES = (
|
|
ROOT / "AGENTS.md",
|
|
ROOT / "docs" / "SOLVER_AGENT_DESIGN.md",
|
|
)
|
|
|
|
EXPECTED_AGENT_STAGE_MAPPING = (
|
|
("requirements", "requirement-agent", ("requirements.md",)),
|
|
("research", "research-agent", ("research.md",)),
|
|
("formulation", "formulation-agent", ("formulation.md",)),
|
|
(
|
|
"numerical-review",
|
|
"numerical-review-agent",
|
|
("numerical-review.md", "reference-model.md"),
|
|
),
|
|
("io-definition", "io-definition-agent", ("io.md",)),
|
|
(
|
|
"implementation-planning",
|
|
"implementation-planning-agent",
|
|
("implementation-plan.md",),
|
|
),
|
|
(
|
|
"implementation",
|
|
"implementation-agent",
|
|
("implementation-report.md", "build-test.md", "reference-comparison.md"),
|
|
),
|
|
("physics-evaluation", "physics-evaluation-agent", ("physics-evaluation.md",)),
|
|
("release-readiness", "release-agent", ("release.md",)),
|
|
)
|
|
|
|
CANONICAL_FEATURE_DIRS = (
|
|
ROOT / "docs" / "linear-static-3d-euler-beam",
|
|
ROOT / "docs" / "linear-static-mitc4-shell",
|
|
)
|
|
|
|
# Plans and specs intentionally retain deleted source paths as migration history.
|
|
HISTORICAL_PATH_EXEMPT_DIRS = (
|
|
ROOT / "docs" / "superpowers" / "plans",
|
|
ROOT / "docs" / "superpowers" / "specs",
|
|
)
|
|
|
|
LEGACY_README_PATTERN = re.compile(
|
|
r"docs/(?:coordination|requirements|research|formulations|numerical-reviews|"
|
|
r"io-definitions|reference-models|implementation-plans|build-test-reports|"
|
|
r"corrections|reference-verifications|physics-evaluations|releases)/README\.md",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
EXPECTED_SKILL_FRONTMATTER_KEYS = ("name", "description")
|
|
EXPECTED_SKILL_INTERFACE_KEYS = (
|
|
"display_name",
|
|
"short_description",
|
|
"default_prompt",
|
|
)
|
|
|
|
RETIRED_TOKENS = (
|
|
"reference-model-agent",
|
|
"reference model agent",
|
|
"build-test-executor-agent",
|
|
"build/test executor agent",
|
|
"reference-verification-agent",
|
|
"reference verification agent",
|
|
"fesa-reference-models",
|
|
"fesa-reference-comparison",
|
|
"docs/coordination/",
|
|
"docs/requirements/",
|
|
"docs/research/",
|
|
"docs/formulations/",
|
|
"docs/numerical-reviews/",
|
|
"docs/io-definitions/",
|
|
"docs/reference-models/",
|
|
"docs/implementation-plans/",
|
|
"docs/build-test-reports/",
|
|
"docs/corrections/",
|
|
"docs/reference-verifications/",
|
|
"docs/physics-evaluations/",
|
|
"docs/releases/",
|
|
)
|
|
|
|
|
|
def read(path: Path) -> str:
|
|
return path.read_text(encoding="utf-8")
|
|
|
|
|
|
def live_contract_text() -> str:
|
|
paths = list(LIVE_CONTRACT_FILES)
|
|
paths.extend(sorted(AGENT_DIR.glob("*.toml")))
|
|
paths.extend(sorted(SKILL_DIR.glob("*/SKILL.md")))
|
|
return "\n".join(read(path) for path in paths)
|
|
|
|
|
|
def markdown_stage_mapping(path: Path) -> tuple[tuple[str, str, tuple[str, ...]], ...]:
|
|
heading = (
|
|
"## Agent/Skill Workflow"
|
|
if path.name == "AGENTS.md"
|
|
else "## 4. 단계와 소유권"
|
|
)
|
|
section = read(path).split(heading, maxsplit=1)[1].split("\n## ", maxsplit=1)[0]
|
|
rows = []
|
|
for line in section.splitlines():
|
|
if not line.lstrip().startswith("|"):
|
|
continue
|
|
columns = [column.strip().replace("`", "") for column in line.strip().split("|")[1:-1]]
|
|
if len(columns) < 3:
|
|
continue
|
|
stage = re.sub(r"^\d+\.\s+", "", columns[0])
|
|
if re.fullmatch(r"[a-z]+(?:-[a-z]+)*", stage):
|
|
reports = tuple(re.findall(r"\b[\w-]+\.md\b", columns[-1]))
|
|
rows.append((stage, columns[1], reports))
|
|
return tuple(rows)
|
|
|
|
|
|
def operational_markdown_paths() -> tuple[Path, ...]:
|
|
paths = [
|
|
path
|
|
for feature_dir in CANONICAL_FEATURE_DIRS
|
|
for path in sorted(feature_dir.glob("*.md"))
|
|
]
|
|
paths.extend(sorted(ROOT.glob("phases/**/step*.md")))
|
|
return tuple(paths)
|
|
|
|
|
|
def parse_skill_frontmatter(path: Path) -> dict[str, str]:
|
|
lines = read(path).splitlines()
|
|
assert lines and lines[0] == "---", path
|
|
assert "---" in lines[1:], path
|
|
closing_index = lines.index("---", 1)
|
|
field_lines = lines[1:closing_index]
|
|
assert len(field_lines) == len(EXPECTED_SKILL_FRONTMATTER_KEYS), path
|
|
|
|
parsed = {}
|
|
for line in field_lines:
|
|
match = re.fullmatch(r"([a-z_]+): ([^\s].*)", line)
|
|
assert match is not None, (path, line)
|
|
key, value = match.groups()
|
|
assert key not in parsed, (path, key)
|
|
parsed[key] = value
|
|
|
|
assert tuple(parsed) == EXPECTED_SKILL_FRONTMATTER_KEYS, path
|
|
assert re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", parsed["name"]), path
|
|
assert parsed["description"].startswith("Use when "), path
|
|
return parsed
|
|
|
|
|
|
def parse_skill_interface(path: Path) -> dict[str, str]:
|
|
lines = [line for line in read(path).splitlines() if line]
|
|
assert lines and lines[0] == "interface:", path
|
|
assert len(lines) == len(EXPECTED_SKILL_INTERFACE_KEYS) + 1, path
|
|
|
|
parsed = {}
|
|
for line in lines[1:]:
|
|
match = re.fullmatch(r' ([a-z_]+): ("(?:[^"\\]|\\.)*")', line)
|
|
assert match is not None, (path, line)
|
|
key, quoted_value = match.groups()
|
|
assert key not in parsed, (path, key)
|
|
parsed[key] = json.loads(quoted_value)
|
|
|
|
assert tuple(parsed) == EXPECTED_SKILL_INTERFACE_KEYS, path
|
|
assert all(parsed.values()), path
|
|
return parsed
|
|
|
|
|
|
def test_agent_inventory_is_consolidated():
|
|
actual = {path.stem for path in AGENT_DIR.glob("*.toml")}
|
|
assert actual == EXPECTED_AGENTS
|
|
|
|
|
|
def test_project_skill_inventory_is_consolidated():
|
|
actual = {
|
|
path.name
|
|
for path in SKILL_DIR.iterdir()
|
|
if path.is_dir() and (path / "SKILL.md").is_file()
|
|
}
|
|
assert actual == EXPECTED_SKILLS
|
|
|
|
|
|
def test_existing_feature_artifacts_are_bundled():
|
|
docs_dir = ROOT / "docs"
|
|
for legacy_dir in LEGACY_OUTPUT_DIRS:
|
|
assert not (docs_dir / legacy_dir).exists(), legacy_dir
|
|
|
|
for feature_id, expected_files in FEATURE_FILES.items():
|
|
feature_dir = docs_dir / feature_id
|
|
actual_files = {path.name for path in feature_dir.glob("*.md")}
|
|
# Preserve historical artifacts while allowing Feature management and linked details.
|
|
assert expected_files <= actual_files
|
|
|
|
|
|
def test_numerical_review_absorbs_reference_model_contract():
|
|
assert not (AGENT_DIR / "reference-model-agent.toml").exists()
|
|
assert not (SKILL_DIR / "fesa-reference-models").exists()
|
|
text = read(SKILL_DIR / "fem-numerical-review" / "SKILL.md")
|
|
outputs = set(re.findall(r"docs/<feature-id>/([\w-]+\.md)", text))
|
|
assert {"numerical-review.md", "reference-model.md"} <= outputs
|
|
|
|
|
|
def test_implementation_absorbs_build_test_and_reference_comparison():
|
|
assert not (AGENT_DIR / "build-test-executor-agent.toml").exists()
|
|
assert not (AGENT_DIR / "reference-verification-agent.toml").exists()
|
|
assert not (SKILL_DIR / "fesa-reference-comparison").exists()
|
|
text = read(SKILL_DIR / "fem-implementation-tdd" / "SKILL.md")
|
|
outputs = set(re.findall(r"docs/<feature-id>/([\w-]+\.md)", text))
|
|
assert {
|
|
"implementation-plan.md",
|
|
"implementation-report.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"corrections.md",
|
|
} <= outputs
|
|
|
|
|
|
def test_implementation_verification_reference_is_resolvable():
|
|
skill = SKILL_DIR / "fem-implementation-tdd" / "SKILL.md"
|
|
links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", read(skill))
|
|
assert "references/toolchain-and-verification.md" in links
|
|
for link in links:
|
|
assert (skill.parent / link).is_file(), link
|
|
|
|
|
|
def test_p_agent_001_implementation_guidance_resolves_coding_style_contract():
|
|
parsed = tomllib.loads(read(AGENT_DIR / "implementation-agent.toml"))
|
|
instructions = parsed["developer_instructions"]
|
|
assert "AGENTS.md" in instructions
|
|
project_links = re.findall(r"\[[^\]]+\]\(([^)]+)\)", read(ROOT / "AGENTS.md"))
|
|
assert "docs/CODINGSTYLE.md" in project_links
|
|
assert (ROOT / "docs/CODINGSTYLE.md").is_file()
|
|
|
|
|
|
def test_agent_profiles_reference_shared_authority_and_feature_context():
|
|
for path in sorted(AGENT_DIR.glob("*.toml")):
|
|
instructions = tomllib.loads(read(path))["developer_instructions"]
|
|
for document in (
|
|
"AGENTS.md",
|
|
"docs/SOLVER_AGENT_DESIGN.md",
|
|
"docs/FEATURE_DOCUMENTATION.md",
|
|
"FEATURED_AGENTS.md",
|
|
"PRD.md",
|
|
"ADR.md",
|
|
):
|
|
assert document in instructions, (path.name, document)
|
|
for skill_name in re.findall(r"\$([a-z0-9-]+)", instructions):
|
|
candidates = (
|
|
SKILL_DIR / skill_name / "SKILL.md",
|
|
ROOT / ".agents/skills" / skill_name / "SKILL.md",
|
|
)
|
|
assert any(candidate.is_file() for candidate in candidates), (path, skill_name)
|
|
|
|
|
|
def test_live_contracts_have_no_retired_names_or_paths():
|
|
text = live_contract_text().lower()
|
|
assert "docs/<feature-id>/" in text
|
|
for token in RETIRED_TOKENS:
|
|
assert token not in text, token
|
|
|
|
|
|
def test_live_guidance_declares_consistent_ordered_nine_stage_workflow():
|
|
for path in LIVE_CONTRACT_FILES:
|
|
text = read(path)
|
|
assert "docs/<feature-id>/" in text, path
|
|
|
|
for path in AGENT_STAGE_GUIDANCE_FILES:
|
|
assert markdown_stage_mapping(path) == EXPECTED_AGENT_STAGE_MAPPING, path
|
|
|
|
|
|
def test_operational_documents_do_not_use_deleted_stage_readmes():
|
|
paths = operational_markdown_paths()
|
|
assert paths
|
|
for path in paths:
|
|
assert not any(path.is_relative_to(directory) for directory in HISTORICAL_PATH_EXEMPT_DIRS)
|
|
match = LEGACY_README_PATTERN.search(read(path))
|
|
assert match is None, (path, match.group(0) if match else None)
|
|
|
|
|
|
def test_mitc4_live_authority_links_use_feature_bundle():
|
|
text = read(ROOT / "docs" / "MITC4_SUPP.md")
|
|
authority_section = text.split("라인 참조", maxsplit=1)[0]
|
|
|
|
for path in (
|
|
"docs/linear-static-mitc4-shell/requirements.md",
|
|
"docs/linear-static-mitc4-shell/formulation.md",
|
|
"docs/linear-static-mitc4-shell/io.md",
|
|
"docs/linear-static-mitc4-shell/reference-model.md",
|
|
"docs/linear-static-mitc4-shell/numerical-review.md",
|
|
"docs/linear-static-mitc4-shell/release.md",
|
|
):
|
|
assert path in authority_section
|
|
|
|
for legacy_path in (
|
|
"docs/requirements/linear-static-mitc4-shell.md",
|
|
"docs/formulations/mitc4-shell-formulation.md",
|
|
"docs/io-definitions/linear-static-mitc4-shell-io.md",
|
|
"docs/reference-models/linear-static-mitc4-shell-reference-models.md",
|
|
"docs/numerical-reviews/linear-static-mitc4-shell-review.md",
|
|
"docs/releases/linear-static-mitc4-shell-release.md",
|
|
):
|
|
assert legacy_path not in authority_section
|
|
|
|
|
|
def test_agent_toml_and_limited_skill_yaml_schemas_are_parseable():
|
|
for path in sorted(AGENT_DIR.glob("*.toml")):
|
|
parsed = tomllib.loads(read(path))
|
|
assert parsed["name"] == path.stem
|
|
assert parsed["model_reasoning_effort"] == "xhigh"
|
|
assert parsed["sandbox_mode"] == "workspace-write"
|
|
assert parsed["developer_instructions"].strip(), path
|
|
|
|
for skill_name in EXPECTED_SKILLS:
|
|
skill_dir = SKILL_DIR / skill_name
|
|
frontmatter = parse_skill_frontmatter(skill_dir / "SKILL.md")
|
|
interface = parse_skill_interface(skill_dir / "agents" / "openai.yaml")
|
|
assert frontmatter["name"] == skill_name
|
|
assert f"${skill_name}" in interface["default_prompt"]
|
|
|
|
|
|
@pytest.mark.parametrize("document", [".agents/skills/harness/SKILL.md", "docs/HARNESS_WORKFLOW.md"])
|
|
def test_documented_phase_example_passes_executor_preflight(step_runtime, document):
|
|
"""Exercise the documented schema with its real consumer, not prose matching."""
|
|
from scripts import execute
|
|
|
|
examples = [json.loads(block) for block in re.findall(r"```json\s*\n(.*?)\n```", read(ROOT / document), re.DOTALL)]
|
|
phase = next(example for example in examples if "steps" in example)
|
|
phase["project"] = "Harness Tests"
|
|
phase["phase"] = "runtime"
|
|
phase["steps"][0]["name"] = "runtime"
|
|
runtime = step_runtime
|
|
runtime.executor._write_json(runtime.index_file, phase)
|
|
executor = execute.StepExecutor("runtime")
|
|
executor._preflight_repository()
|
|
assert runtime.calls == []
|