KOKO\Mimi
400 lines
13 KiB
Python
400 lines
13 KiB
Python
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tomllib
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
AGENT_DIR = ROOT / ".codex" / "agents"
|
|
SKILL_DIR = ROOT / ".codex" / "skills"
|
|
|
|
EXPECTED_AGENTS = {
|
|
"coordinator-agent",
|
|
"correction-agent",
|
|
"formulation-agent",
|
|
"implementation-agent",
|
|
"implementation-planning-agent",
|
|
"io-definition-agent",
|
|
"numerical-review-agent",
|
|
"physics-evaluation-agent",
|
|
"release-agent",
|
|
"requirement-agent",
|
|
"research-agent",
|
|
}
|
|
|
|
EXPECTED_SKILLS = {
|
|
"fem-theory-query",
|
|
"fesa-cpp-msvc-tdd",
|
|
"fesa-formulation-spec",
|
|
"fesa-io-contract",
|
|
"fesa-numerical-review",
|
|
"fesa-physics-sanity",
|
|
"fesa-release-readiness",
|
|
"fesa-requirements-baseline",
|
|
"fesa-research-evidence",
|
|
}
|
|
|
|
LEGACY_OUTPUT_DIRS = {
|
|
"coordination",
|
|
"requirements",
|
|
"research",
|
|
"formulations",
|
|
"numerical-reviews",
|
|
"io-definitions",
|
|
"reference-models",
|
|
"implementation-plans",
|
|
"build-test-reports",
|
|
"corrections",
|
|
"reference-verifications",
|
|
"physics-evaluations",
|
|
"releases",
|
|
}
|
|
|
|
FEATURE_FILES = {
|
|
"linear-static-3d-euler-beam": {
|
|
"coordination.md",
|
|
"requirements.md",
|
|
"research.md",
|
|
"formulation.md",
|
|
"numerical-review.md",
|
|
"reference-model.md",
|
|
"io.md",
|
|
"implementation-plan.md",
|
|
"implementation-report.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"physics-evaluation.md",
|
|
"release.md",
|
|
},
|
|
"linear-static-mitc4-shell": {
|
|
"coordination.md",
|
|
"requirements.md",
|
|
"research.md",
|
|
"formulation.md",
|
|
"numerical-review.md",
|
|
"reference-model.md",
|
|
"io.md",
|
|
"implementation-plan.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"physics-evaluation.md",
|
|
"release.md",
|
|
},
|
|
}
|
|
|
|
LIVE_CONTRACT_FILES = (
|
|
ROOT / "AGENTS.md",
|
|
ROOT / "docs" / "SOLVER_AGENT_DESIGN.md",
|
|
ROOT / "docs" / "SOLVER_SKILL_DESIGN.md",
|
|
)
|
|
|
|
AGENT_STAGE_GUIDANCE_FILES = (
|
|
ROOT / "AGENTS.md",
|
|
ROOT / "docs" / "SOLVER_AGENT_DESIGN.md",
|
|
)
|
|
|
|
EXPECTED_AGENT_STAGE_MAPPING = (
|
|
("1. 요구조건", "requirement-agent", "requirements.md"),
|
|
("2. 연구", "research-agent", "research.md"),
|
|
("3. 정식화", "formulation-agent", "formulation.md"),
|
|
(
|
|
"4. 수치 검토 + reference model 계약",
|
|
"numerical-review-agent",
|
|
"numerical-review.md, reference-model.md",
|
|
),
|
|
("5. I/O 정의", "io-definition-agent", "io.md"),
|
|
(
|
|
"6. 구현 계획 + C++ 구현 + build/test + reference comparison",
|
|
"implementation-planning-agent, implementation-agent",
|
|
"implementation-plan.md, implementation-report.md, build-test.md, reference-comparison.md",
|
|
),
|
|
("7. 물리 검토", "physics-evaluation-agent", "physics-evaluation.md"),
|
|
("8. 배포 준비", "release-agent", "release.md"),
|
|
)
|
|
|
|
CANONICAL_FEATURE_DIRS = (
|
|
ROOT / "docs" / "linear-static-3d-euler-beam",
|
|
ROOT / "docs" / "linear-static-mitc4-shell",
|
|
)
|
|
|
|
# Plans and specs intentionally retain deleted source paths as migration history.
|
|
HISTORICAL_PATH_EXEMPT_DIRS = (
|
|
ROOT / "docs" / "superpowers" / "plans",
|
|
ROOT / "docs" / "superpowers" / "specs",
|
|
)
|
|
|
|
LEGACY_README_PATTERN = re.compile(
|
|
r"docs/(?:coordination|requirements|research|formulations|numerical-reviews|"
|
|
r"io-definitions|reference-models|implementation-plans|build-test-reports|"
|
|
r"corrections|reference-verifications|physics-evaluations|releases)/README\.md",
|
|
re.IGNORECASE,
|
|
)
|
|
|
|
EXPECTED_SKILL_FRONTMATTER_KEYS = ("name", "description")
|
|
EXPECTED_SKILL_INTERFACE_KEYS = (
|
|
"display_name",
|
|
"short_description",
|
|
"default_prompt",
|
|
)
|
|
|
|
RETIRED_TOKENS = (
|
|
"reference-model-agent",
|
|
"reference model agent",
|
|
"build-test-executor-agent",
|
|
"build/test executor agent",
|
|
"reference-verification-agent",
|
|
"reference verification agent",
|
|
"fesa-reference-models",
|
|
"fesa-reference-comparison",
|
|
"docs/coordination/",
|
|
"docs/requirements/",
|
|
"docs/research/",
|
|
"docs/formulations/",
|
|
"docs/numerical-reviews/",
|
|
"docs/io-definitions/",
|
|
"docs/reference-models/",
|
|
"docs/implementation-plans/",
|
|
"docs/build-test-reports/",
|
|
"docs/corrections/",
|
|
"docs/reference-verifications/",
|
|
"docs/physics-evaluations/",
|
|
"docs/releases/",
|
|
)
|
|
|
|
|
|
def read(path: Path) -> str:
|
|
return path.read_text(encoding="utf-8")
|
|
|
|
|
|
def live_contract_text() -> str:
|
|
paths = list(LIVE_CONTRACT_FILES)
|
|
paths.extend(sorted(AGENT_DIR.glob("*.toml")))
|
|
paths.extend(sorted(SKILL_DIR.glob("*/SKILL.md")))
|
|
return "\n".join(read(path) for path in paths)
|
|
|
|
|
|
def markdown_stage_mapping(path: Path) -> tuple[tuple[str, str, str], ...]:
|
|
rows = []
|
|
for line in read(path).splitlines():
|
|
columns = [column.strip().replace("`", "") for column in line.strip().split("|")[1:-1]]
|
|
if len(columns) >= 4 and re.fullmatch(r"[1-8]\. .+", columns[0]):
|
|
rows.append((columns[0], columns[1], columns[3]))
|
|
return tuple(rows)
|
|
|
|
|
|
def operational_markdown_paths() -> tuple[Path, ...]:
|
|
paths = [
|
|
path
|
|
for feature_dir in CANONICAL_FEATURE_DIRS
|
|
for path in sorted(feature_dir.glob("*.md"))
|
|
]
|
|
paths.extend(sorted(ROOT.glob("phases/**/step*.md")))
|
|
return tuple(paths)
|
|
|
|
|
|
def parse_skill_frontmatter(path: Path) -> dict[str, str]:
|
|
lines = read(path).splitlines()
|
|
assert lines and lines[0] == "---", path
|
|
assert "---" in lines[1:], path
|
|
closing_index = lines.index("---", 1)
|
|
field_lines = lines[1:closing_index]
|
|
assert len(field_lines) == len(EXPECTED_SKILL_FRONTMATTER_KEYS), path
|
|
|
|
parsed = {}
|
|
for line in field_lines:
|
|
match = re.fullmatch(r"([a-z_]+): ([^\s].*)", line)
|
|
assert match is not None, (path, line)
|
|
key, value = match.groups()
|
|
assert key not in parsed, (path, key)
|
|
parsed[key] = value
|
|
|
|
assert tuple(parsed) == EXPECTED_SKILL_FRONTMATTER_KEYS, path
|
|
assert re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", parsed["name"]), path
|
|
assert parsed["description"].startswith("Use when "), path
|
|
return parsed
|
|
|
|
|
|
def parse_skill_interface(path: Path) -> dict[str, str]:
|
|
lines = [line for line in read(path).splitlines() if line]
|
|
assert lines and lines[0] == "interface:", path
|
|
assert len(lines) == len(EXPECTED_SKILL_INTERFACE_KEYS) + 1, path
|
|
|
|
parsed = {}
|
|
for line in lines[1:]:
|
|
match = re.fullmatch(r' ([a-z_]+): ("(?:[^"\\]|\\.)*")', line)
|
|
assert match is not None, (path, line)
|
|
key, quoted_value = match.groups()
|
|
assert key not in parsed, (path, key)
|
|
parsed[key] = json.loads(quoted_value)
|
|
|
|
assert tuple(parsed) == EXPECTED_SKILL_INTERFACE_KEYS, path
|
|
assert all(parsed.values()), path
|
|
return parsed
|
|
|
|
|
|
def test_agent_inventory_is_consolidated():
|
|
actual = {path.stem for path in AGENT_DIR.glob("*.toml")}
|
|
assert actual == EXPECTED_AGENTS
|
|
|
|
|
|
def test_project_skill_inventory_is_consolidated():
|
|
actual = {
|
|
path.name
|
|
for path in SKILL_DIR.iterdir()
|
|
if path.is_dir() and (path / "SKILL.md").is_file()
|
|
}
|
|
assert actual == EXPECTED_SKILLS
|
|
|
|
|
|
def test_existing_feature_artifacts_are_bundled():
|
|
docs_dir = ROOT / "docs"
|
|
for legacy_dir in LEGACY_OUTPUT_DIRS:
|
|
assert not (docs_dir / legacy_dir).exists(), legacy_dir
|
|
|
|
for feature_id, expected_files in FEATURE_FILES.items():
|
|
feature_dir = docs_dir / feature_id
|
|
actual_files = {path.name for path in feature_dir.glob("*.md")}
|
|
assert actual_files == expected_files
|
|
|
|
|
|
def test_numerical_review_absorbs_reference_model_contract():
|
|
assert not (AGENT_DIR / "reference-model-agent.toml").exists()
|
|
assert not (SKILL_DIR / "fesa-reference-models").exists()
|
|
text = read(SKILL_DIR / "fesa-numerical-review" / "SKILL.md").lower()
|
|
for marker in (
|
|
"numerical-review.md",
|
|
"reference-model.md",
|
|
"reference case inventory",
|
|
"source identity",
|
|
"row precheck",
|
|
"tolerance",
|
|
"pass-for-io-definition",
|
|
):
|
|
assert marker in text
|
|
|
|
|
|
def test_implementation_absorbs_build_test_and_reference_comparison():
|
|
assert not (AGENT_DIR / "build-test-executor-agent.toml").exists()
|
|
assert not (AGENT_DIR / "reference-verification-agent.toml").exists()
|
|
assert not (SKILL_DIR / "fesa-reference-comparison").exists()
|
|
text = read(SKILL_DIR / "fesa-cpp-msvc-tdd" / "SKILL.md").lower()
|
|
for marker in (
|
|
"implementation-report.md",
|
|
"build-test.md",
|
|
"reference-comparison.md",
|
|
"artifact check -> compare -> classify -> report",
|
|
"pass-for-physics-evaluation",
|
|
):
|
|
assert marker in text
|
|
|
|
|
|
def test_implementation_steps_keep_final_comparison_in_a_final_gate():
|
|
paths = (
|
|
AGENT_DIR / "implementation-agent.toml",
|
|
SKILL_DIR / "fesa-cpp-msvc-tdd" / "SKILL.md",
|
|
)
|
|
for path in paths:
|
|
text = read(path).lower()
|
|
assert "final implementation-owned verification step/gate" in text, path
|
|
assert "non-final step" in text, path
|
|
assert "artifact check -> compare -> classify -> report" in text, path
|
|
assert "results.h5" in text, path
|
|
assert "reference-comparison.md" in text, path
|
|
|
|
|
|
def test_p_agent_001_implementation_agent_reads_coding_style_before_cpp_steps():
|
|
parsed = tomllib.loads(read(AGENT_DIR / "implementation-agent.toml"))
|
|
instructions = parsed["developer_instructions"]
|
|
section_match = re.search(
|
|
r"Mandatory global input:\n(?P<section>.*?)(?=\n[A-Z][^\n]+:\n)",
|
|
instructions,
|
|
re.DOTALL,
|
|
)
|
|
|
|
assert section_match is not None
|
|
section = section_match.group("section")
|
|
assert "Before every C++ implementation Step" in section
|
|
assert "read docs/CODINGSTYLE.md" in section
|
|
assert "apply it to production and test code" in section
|
|
assert "Doxygen coverage applies only to production code" in section
|
|
|
|
|
|
def test_agent_hierarchy_is_explicit():
|
|
coordinator = read(AGENT_DIR / "coordinator-agent.toml").lower()
|
|
assert "main agent" in coordinator
|
|
assert "docs/<feature-id>/coordination.md" in coordinator
|
|
assert "worklist" in coordinator
|
|
assert "sub-agent dispatch" in coordinator
|
|
|
|
for path in sorted(AGENT_DIR.glob("*.toml")):
|
|
if path.name == "coordinator-agent.toml":
|
|
continue
|
|
text = read(path).lower()
|
|
assert "sub-agent" in text, path.name
|
|
assert "coordinator agent" in text, path.name
|
|
|
|
|
|
def test_live_contracts_have_no_retired_names_or_paths():
|
|
text = live_contract_text().lower()
|
|
assert "docs/<feature-id>/" in text
|
|
for token in RETIRED_TOKENS:
|
|
assert token not in text, token
|
|
|
|
|
|
def test_live_guidance_declares_exact_ordered_eight_stage_workflow():
|
|
for path in LIVE_CONTRACT_FILES:
|
|
text = read(path)
|
|
assert "8단계" in text, path
|
|
assert "docs/<feature-id>/" in text, path
|
|
|
|
for path in AGENT_STAGE_GUIDANCE_FILES:
|
|
assert markdown_stage_mapping(path) == EXPECTED_AGENT_STAGE_MAPPING, path
|
|
|
|
|
|
def test_operational_documents_do_not_use_deleted_stage_readmes():
|
|
paths = operational_markdown_paths()
|
|
assert paths
|
|
for path in paths:
|
|
assert not any(path.is_relative_to(directory) for directory in HISTORICAL_PATH_EXEMPT_DIRS)
|
|
match = LEGACY_README_PATTERN.search(read(path))
|
|
assert match is None, (path, match.group(0) if match else None)
|
|
|
|
|
|
def test_mitc4_live_authority_links_use_feature_bundle():
|
|
text = read(ROOT / "docs" / "MITC4_SUPP.md")
|
|
authority_section = text.split("라인 참조", maxsplit=1)[0]
|
|
|
|
for path in (
|
|
"docs/linear-static-mitc4-shell/requirements.md",
|
|
"docs/linear-static-mitc4-shell/formulation.md",
|
|
"docs/linear-static-mitc4-shell/io.md",
|
|
"docs/linear-static-mitc4-shell/reference-model.md",
|
|
"docs/linear-static-mitc4-shell/numerical-review.md",
|
|
"docs/linear-static-mitc4-shell/release.md",
|
|
):
|
|
assert path in authority_section
|
|
|
|
for legacy_path in (
|
|
"docs/requirements/linear-static-mitc4-shell.md",
|
|
"docs/formulations/mitc4-shell-formulation.md",
|
|
"docs/io-definitions/linear-static-mitc4-shell-io.md",
|
|
"docs/reference-models/linear-static-mitc4-shell-reference-models.md",
|
|
"docs/numerical-reviews/linear-static-mitc4-shell-review.md",
|
|
"docs/releases/linear-static-mitc4-shell-release.md",
|
|
):
|
|
assert legacy_path not in authority_section
|
|
|
|
|
|
def test_agent_toml_and_limited_skill_yaml_schemas_are_parseable():
|
|
for path in sorted(AGENT_DIR.glob("*.toml")):
|
|
parsed = tomllib.loads(read(path))
|
|
assert parsed["name"] == path.stem
|
|
assert parsed["model_reasoning_effort"] == "extra high"
|
|
|
|
for skill_name in EXPECTED_SKILLS:
|
|
skill_dir = SKILL_DIR / skill_name
|
|
frontmatter = parse_skill_frontmatter(skill_dir / "SKILL.md")
|
|
interface = parse_skill_interface(skill_dir / "agents" / "openai.yaml")
|
|
assert frontmatter["name"] == skill_name
|
|
assert f"${skill_name}" in interface["default_prompt"]
|