Add Bmad-Method files and some brand-design artifacts!
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Guard against drift between the embedded prompt-quality-canon copies.
|
||||
|
||||
The canon is embedded in three places — workflow-builder references,
|
||||
agent-builder references, and the agent-builder asset emitted into built
|
||||
agents — with no in-file sync note, because the loaded files are LLM-facing
|
||||
and a maintenance comment there is paid on every load. This test is the
|
||||
sync mechanism instead: all three copies must be byte-identical.
|
||||
Run with: python3 -m pytest test_canon_sync.py
|
||||
(or plain `python3 test_canon_sync.py` for a lightweight self-check).
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SKILLS_DIR = Path(__file__).resolve().parents[3]
|
||||
|
||||
CANON_COPIES = [
|
||||
SKILLS_DIR / "bmad-workflow-builder" / "references" / "prompt-quality-canon.md",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "references" / "prompt-quality-canon.md",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "assets" / "prompt-quality-canon.md",
|
||||
]
|
||||
|
||||
|
||||
def test_all_copies_exist():
|
||||
missing = [str(p) for p in CANON_COPIES if not p.is_file()]
|
||||
assert not missing, f"canon copy missing: {missing}"
|
||||
|
||||
|
||||
def test_all_copies_identical():
|
||||
contents = {p: p.read_bytes() for p in CANON_COPIES if p.is_file()}
|
||||
reference = CANON_COPIES[0]
|
||||
diverged = [
|
||||
str(p)
|
||||
for p, body in contents.items()
|
||||
if body != contents.get(reference)
|
||||
]
|
||||
assert not diverged, (
|
||||
"canon copies have drifted from "
|
||||
f"{reference}: {diverged} — sync all copies together"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_all_copies_exist()
|
||||
test_all_copies_identical()
|
||||
print(f"ok: {len(CANON_COPIES)} canon copies present and identical")
|
||||
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for count_tokens.py.
|
||||
|
||||
Covers the output schema, the tiktoken path and the forced-fallback path
|
||||
agreeing within tolerance, the CLI over a file and over stdin, and argument
|
||||
guards. Run with: python3 -m pytest test_count_tokens.py
|
||||
(or plain `python3 test_count_tokens.py` to run a lightweight self-check).
|
||||
"""
|
||||
import builtins
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent.parent / "count_tokens.py"
|
||||
|
||||
SAMPLE = (
|
||||
"The builder is platform-agnostic. Nothing assumes a single runtime, and no "
|
||||
"model list is ever hardcoded. Token counts replace line counts as the one "
|
||||
"length metric, with a chars-over-four fallback when tiktoken is absent.\n"
|
||||
) * 8
|
||||
|
||||
|
||||
def _load_module():
|
||||
spec = importlib.util.spec_from_file_location("count_tokens", SCRIPT)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def test_tiktoken_path():
|
||||
mod = _load_module()
|
||||
try:
|
||||
import tiktoken # noqa: F401
|
||||
except Exception:
|
||||
# No tiktoken in this interpreter; the real path can't be exercised here.
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
assert method == "fallback"
|
||||
assert tokens == len(SAMPLE) // 4
|
||||
return
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
assert method == "tiktoken"
|
||||
assert isinstance(tokens, int)
|
||||
assert tokens > 0
|
||||
|
||||
|
||||
def test_fallback_path_when_import_blocked():
|
||||
"""Force the import of tiktoken to fail and confirm the fallback fires."""
|
||||
mod = _load_module()
|
||||
real_import = builtins.__import__
|
||||
|
||||
def blocked_import(name, *args, **kwargs):
|
||||
if name == "tiktoken" or name.startswith("tiktoken."):
|
||||
raise ImportError("blocked for test")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
builtins.__import__ = blocked_import
|
||||
try:
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
finally:
|
||||
builtins.__import__ = real_import
|
||||
|
||||
assert method == "fallback"
|
||||
assert tokens == len(SAMPLE) // 4
|
||||
|
||||
|
||||
def test_paths_agree_within_tolerance():
|
||||
"""tiktoken and chars//4 should be in the same order of magnitude.
|
||||
|
||||
Skipped when tiktoken is not installed (nothing to compare against).
|
||||
"""
|
||||
mod = _load_module()
|
||||
try:
|
||||
import tiktoken # noqa: F401
|
||||
except Exception:
|
||||
return
|
||||
|
||||
real_tokens, real_method = mod.count_tokens(SAMPLE)
|
||||
assert real_method == "tiktoken"
|
||||
|
||||
fallback_tokens = len(SAMPLE) // 4
|
||||
|
||||
# The chars//4 heuristic is a rough proxy; require it within +/-50% of the
|
||||
# real count so the fallback stays a usable budget gate, not a wild guess.
|
||||
lower = real_tokens * 0.5
|
||||
upper = real_tokens * 1.5
|
||||
assert lower <= fallback_tokens <= upper, (
|
||||
f"fallback {fallback_tokens} not within 50% of tiktoken {real_tokens}"
|
||||
)
|
||||
|
||||
|
||||
def test_cli_file_output_schema(tmp_path):
|
||||
f = tmp_path / "sample.md"
|
||||
f.write_text(SAMPLE, encoding="utf-8")
|
||||
out = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), str(f)],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
assert set(data.keys()) == {"tokens", "method"}
|
||||
assert isinstance(data["tokens"], int)
|
||||
assert data["method"] in ("tiktoken", "fallback")
|
||||
assert data["tokens"] > 0
|
||||
|
||||
|
||||
def test_cli_stdin_output_schema():
|
||||
out = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--stdin"],
|
||||
input=SAMPLE, capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
assert set(data.keys()) == {"tokens", "method"}
|
||||
assert isinstance(data["tokens"], int)
|
||||
assert data["method"] in ("tiktoken", "fallback")
|
||||
|
||||
|
||||
def test_cli_file_and_stdin_agree():
|
||||
"""The CLI over a file and over stdin produce the same count for same text."""
|
||||
import tempfile, os
|
||||
fd, name = tempfile.mkstemp(suffix=".md")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(SAMPLE)
|
||||
file_out = json.loads(subprocess.run(
|
||||
[sys.executable, str(SCRIPT), name],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout)
|
||||
finally:
|
||||
os.unlink(name)
|
||||
stdin_out = json.loads(subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--stdin"],
|
||||
input=SAMPLE, capture_output=True, text=True, check=True,
|
||||
).stdout)
|
||||
assert file_out == stdin_out
|
||||
|
||||
|
||||
def test_cli_requires_an_input():
|
||||
"""No file and no --stdin is a usage error (exit 2 from argparse)."""
|
||||
res = subprocess.run(
|
||||
[sys.executable, str(SCRIPT)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert res.returncode != 0
|
||||
|
||||
|
||||
def _run_all():
|
||||
import tempfile
|
||||
failures = 0
|
||||
tests = [
|
||||
test_tiktoken_path,
|
||||
test_fallback_path_when_import_blocked,
|
||||
test_paths_agree_within_tolerance,
|
||||
test_cli_stdin_output_schema,
|
||||
test_cli_file_and_stdin_agree,
|
||||
test_cli_requires_an_input,
|
||||
]
|
||||
for t in tests:
|
||||
try:
|
||||
t()
|
||||
print(f"PASS {t.__name__}")
|
||||
except AssertionError as e:
|
||||
failures += 1
|
||||
print(f"FAIL {t.__name__}: {e}")
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
print(f"ERROR {t.__name__}: {e}")
|
||||
# tmp_path-based test handled separately
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
try:
|
||||
test_cli_file_output_schema(Path(d))
|
||||
print("PASS test_cli_file_output_schema")
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
print(f"FAIL test_cli_file_output_schema: {e}")
|
||||
return failures
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(1 if _run_all() else 0)
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for scripts/render_report.py — the deterministic report renderer.
|
||||
|
||||
Covers: valid island injection, refusal on malformed JSON, refusal on the
|
||||
placeholder subject, the --md archival rendering, and that both shipped
|
||||
shells carry a parseable placeholder island.
|
||||
Run with: python3 -m pytest test_render_report.py
|
||||
(or plain `python3 test_render_report.py` for a lightweight self-check).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SKILLS_DIR = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = SKILLS_DIR / "bmad-workflow-builder" / "scripts" / "render_report.py"
|
||||
SHELLS = [
|
||||
SKILLS_DIR / "bmad-workflow-builder" / "assets" / "report-shell.html",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "assets" / "report-shell.html",
|
||||
]
|
||||
ISLAND_RE = re.compile(
|
||||
r'<script[^>]*\bid="report-data"[^>]*>(.*?)</script>', re.DOTALL
|
||||
)
|
||||
|
||||
VALID_DATA = {
|
||||
"schema_version": 2,
|
||||
"subject": "skills/example-skill",
|
||||
"generated": "2026-06-10",
|
||||
"verdict": "One ceremony section; otherwise sound.",
|
||||
"grade": "good",
|
||||
"summary": "Solid structure and clean wiring. The main opportunity is one over-scripted reference.",
|
||||
"standards": {
|
||||
"canon": "/abs/skills/bmad-workflow-builder/references/prompt-quality-canon.md",
|
||||
"principles": "/abs/skills/bmad-workflow-builder/references/skill-quality-principles.md",
|
||||
"scripts": "/abs/skills/bmad-workflow-builder/references/script-standards.md",
|
||||
},
|
||||
"themes": [
|
||||
{
|
||||
"title": "Scripted sequences where goals suffice",
|
||||
"root_cause": "Steps are numbered without true ordering dependencies.",
|
||||
"finding_ids": ["leanness-1"],
|
||||
"action": "Replace ordered lists with goal sentences.",
|
||||
}
|
||||
],
|
||||
"strengths": ["Frontmatter and routing map are exemplary."],
|
||||
"recommendations": [
|
||||
{"rank": 1, "action": "De-script the finalize section.", "resolves": ["leanness-1"]}
|
||||
],
|
||||
"findings": [
|
||||
{
|
||||
"id": "leanness-1",
|
||||
"lens": "leanness",
|
||||
"severity": "high",
|
||||
"title": "Numbered finalize steps are decoration",
|
||||
"location": "references/build-process.md:finalize",
|
||||
"evidence": "No step depends on a prior step's output.",
|
||||
"recommendation": "Replace with a single goal sentence.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_render(args):
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *[str(a) for a in args]],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_valid_island_injection():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
findings.write_text(json.dumps(VALID_DATA), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode == 0, result.stderr
|
||||
html = out.read_text(encoding="utf-8")
|
||||
|
||||
match = ISLAND_RE.search(html)
|
||||
assert match, "rendered HTML has no report-data island"
|
||||
island = json.loads(match.group(1))
|
||||
assert island["subject"] == "skills/example-skill"
|
||||
assert island["findings"][0]["id"] == "leanness-1"
|
||||
assert island["standards"]["canon"].endswith("prompt-quality-canon.md")
|
||||
assert "__PLACEHOLDER__" not in match.group(1)
|
||||
|
||||
stdout = json.loads(result.stdout)
|
||||
assert stdout["counts"] == {"critical": 0, "high": 1, "medium": 0, "low": 0}
|
||||
assert stdout["grade"] == "good"
|
||||
|
||||
|
||||
def test_refuses_bad_json():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
findings.write_text("{ this is not json", encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode != 0
|
||||
assert "not valid JSON" in result.stderr
|
||||
assert not out.exists(), "refused render must not write output"
|
||||
|
||||
|
||||
def test_refuses_placeholder_subject():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
data = dict(VALID_DATA, subject="__PLACEHOLDER__")
|
||||
findings.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode != 0
|
||||
assert "placeholder" in result.stderr.lower()
|
||||
assert not out.exists(), "refused render must not write output"
|
||||
|
||||
|
||||
def test_md_output():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
md = tmp / "report.md"
|
||||
findings.write_text(json.dumps(VALID_DATA), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out, "--md", md])
|
||||
assert result.returncode == 0, result.stderr
|
||||
text = md.read_text(encoding="utf-8")
|
||||
assert "# Analysis Report: skills/example-skill" in text
|
||||
assert "**Grade: Good**" in text
|
||||
assert "## Themes" in text
|
||||
assert "Scripted sequences where goals suffice" in text
|
||||
assert "## Strengths" in text
|
||||
assert "## Recommendations" in text
|
||||
assert "### High (1)" in text
|
||||
assert "leanness-1" in text
|
||||
|
||||
|
||||
def test_shipped_shells_carry_placeholder_island():
|
||||
for shell in SHELLS:
|
||||
match = ISLAND_RE.search(shell.read_text(encoding="utf-8"))
|
||||
assert match, f"{shell} has no report-data island"
|
||||
island = json.loads(match.group(1))
|
||||
assert island["subject"] == "__PLACEHOLDER__", (
|
||||
f"{shell} ships a non-placeholder island; a failed injection "
|
||||
"would show its contents as real findings"
|
||||
)
|
||||
assert island["findings"] == []
|
||||
|
||||
|
||||
def test_render_script_copies_identical():
|
||||
other = SKILLS_DIR / "bmad-agent-builder" / "scripts" / "render_report.py"
|
||||
assert SCRIPT.read_bytes() == other.read_bytes(), (
|
||||
"render_report.py copies have drifted between the two builder skills"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_valid_island_injection()
|
||||
test_refuses_bad_json()
|
||||
test_refuses_placeholder_subject()
|
||||
test_md_output()
|
||||
test_shipped_shells_carry_placeholder_island()
|
||||
test_render_script_copies_identical()
|
||||
print("ok: render_report tests passed")
|
||||
Reference in New Issue
Block a user