Add Bmad-Method files and some brand-design artifacts!
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# dependencies = ["tiktoken"]
|
||||
# ///
|
||||
"""count_tokens — the single length metric for skill authoring.
|
||||
|
||||
Token counts replace line counts everywhere in the builder and eval-runner.
|
||||
This script reports the token length of a file or of text piped on stdin, using
|
||||
the tiktoken cl100k_base encoding. When tiktoken is not installed it falls back
|
||||
to a character-based estimate (len(text) // 4) and says so, so the script always
|
||||
runs under a bare python3 even with no third-party packages present.
|
||||
|
||||
Usage:
|
||||
count_tokens.py <file> count the tokens in a file
|
||||
count_tokens.py --stdin count the tokens read from stdin
|
||||
|
||||
Output (one line of JSON on stdout):
|
||||
{"tokens": <int>, "method": "tiktoken"} when tiktoken loaded
|
||||
{"tokens": <int>, "method": "fallback"} when it fell back to chars // 4
|
||||
|
||||
Budgets this feeds: SKILL.md ~1500-2500, multi-branch reference ~4500,
|
||||
single-purpose reference ~9000.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
|
||||
ENCODING = "cl100k_base"
|
||||
|
||||
|
||||
def count_tokens(text: str) -> tuple[int, str]:
|
||||
"""Return (token_count, method).
|
||||
|
||||
Tries tiktoken's cl100k_base encoding first. If tiktoken cannot be imported
|
||||
or initialized, estimates with len(text) // 4 and reports method "fallback".
|
||||
"""
|
||||
try:
|
||||
import tiktoken
|
||||
except Exception:
|
||||
return len(text) // 4, "fallback"
|
||||
try:
|
||||
enc = tiktoken.get_encoding(ENCODING)
|
||||
except Exception:
|
||||
return len(text) // 4, "fallback"
|
||||
return len(enc.encode(text)), "tiktoken"
|
||||
|
||||
|
||||
def read_input(args) -> str:
|
||||
if args.stdin:
|
||||
return sys.stdin.read()
|
||||
with open(args.file, encoding="utf-8") as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
p.add_argument("file", nargs="?", help="path to the file to count")
|
||||
p.add_argument("--stdin", action="store_true", help="read text from stdin instead of a file")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if not args.stdin and not args.file:
|
||||
p.error("provide a file path or --stdin")
|
||||
if args.stdin and args.file:
|
||||
p.error("provide either a file path or --stdin, not both")
|
||||
|
||||
text = read_input(args)
|
||||
tokens, method = count_tokens(text)
|
||||
print(json.dumps({"tokens": tokens, "method": method}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# ///
|
||||
"""init_skill — deterministic scaffolder for a new skill.
|
||||
|
||||
Creates the skill directory and writes SKILL.md from the builder's template, which
|
||||
carries the embedded archetype guidance and the delete-when-done marker. The name
|
||||
is normalized to hyphen-case and capped at 64 chars. Only the resource directories
|
||||
the build flow asked for are stubbed, so the skill starts as small as it can. A
|
||||
customize.toml is emitted only when customization was accepted, never by default.
|
||||
|
||||
This script does the mechanical scaffolding so the model spends its turns on the
|
||||
content, not on mkdir and string substitution.
|
||||
|
||||
Usage:
|
||||
init_skill.py --name "My New Skill" --dest /path/to/skills
|
||||
init_skill.py --name foo --dest DIR --dirs references,scripts,assets
|
||||
init_skill.py --name foo --dest DIR --customizable
|
||||
init_skill.py --name foo --dest DIR \
|
||||
--template /abs/SKILL-template.md --customize-template /abs/customize-template.toml
|
||||
|
||||
Output: one JSON object on stdout describing what was created.
|
||||
Exit code 0 on success, 1 on failure (e.g. the target already exists).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
KNOWN_DIRS = ("references", "scripts", "assets", "agents")
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
DEFAULT_TEMPLATE = SCRIPT_DIR.parent / "assets" / "SKILL-template.md"
|
||||
DEFAULT_CUSTOMIZE = SCRIPT_DIR.parent / "assets" / "customize-template.toml"
|
||||
|
||||
|
||||
def normalize_name(raw: str, max_len: int = 64) -> str:
|
||||
"""Lowercase, collapse non-alphanumerics to single hyphens, trim, cap at max_len."""
|
||||
s = raw.strip().lower()
|
||||
s = re.sub(r"[^a-z0-9]+", "-", s)
|
||||
s = s.strip("-")
|
||||
if len(s) > max_len:
|
||||
s = s[:max_len].rstrip("-")
|
||||
return s
|
||||
|
||||
|
||||
def fill_template(template: str, skill_name: str) -> str:
|
||||
return template.replace("{skill-name}", skill_name)
|
||||
|
||||
|
||||
def scaffold(args) -> dict:
|
||||
skill_name = normalize_name(args.name)
|
||||
if not skill_name:
|
||||
raise ValueError(f"name {args.name!r} normalized to an empty string")
|
||||
|
||||
skill_dir = Path(args.dest) / skill_name
|
||||
if skill_dir.exists():
|
||||
raise FileExistsError(f"{skill_dir} already exists")
|
||||
|
||||
template_path = Path(args.template) if args.template else DEFAULT_TEMPLATE
|
||||
if not template_path.is_file():
|
||||
raise FileNotFoundError(f"template not found: {template_path}")
|
||||
|
||||
requested = []
|
||||
for d in (args.dirs or "").split(","):
|
||||
d = d.strip()
|
||||
if not d:
|
||||
continue
|
||||
if d not in KNOWN_DIRS:
|
||||
raise ValueError(f"unknown resource dir {d!r}; known: {', '.join(KNOWN_DIRS)}")
|
||||
requested.append(d)
|
||||
|
||||
skill_dir.mkdir(parents=True)
|
||||
created = [str(skill_dir)]
|
||||
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text(fill_template(template_path.read_text(encoding="utf-8"), skill_name), encoding="utf-8")
|
||||
created.append(str(skill_md))
|
||||
|
||||
for d in requested:
|
||||
sub = skill_dir / d
|
||||
sub.mkdir()
|
||||
created.append(str(sub))
|
||||
|
||||
customize_emitted = False
|
||||
if args.customizable:
|
||||
ct_path = Path(args.customize_template) if args.customize_template else DEFAULT_CUSTOMIZE
|
||||
if not ct_path.is_file():
|
||||
raise FileNotFoundError(f"customize template not found: {ct_path}")
|
||||
target = skill_dir / "customize.toml"
|
||||
target.write_text(
|
||||
ct_path.read_text(encoding="utf-8").replace("{skill-name}", skill_name),
|
||||
encoding="utf-8",
|
||||
)
|
||||
created.append(str(target))
|
||||
customize_emitted = True
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"skill_name": skill_name,
|
||||
"skill_dir": str(skill_dir),
|
||||
"dirs_stubbed": requested,
|
||||
"customize_toml": customize_emitted,
|
||||
"created": created,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description="Deterministic scaffolder for a new skill")
|
||||
p.add_argument("--name", required=True, help="raw skill name; normalized to hyphen-case <=64")
|
||||
p.add_argument("--dest", required=True, help="parent directory the skill folder is created under")
|
||||
p.add_argument("--dirs", default="", help="comma-separated resource dirs to stub (references,scripts,assets,agents)")
|
||||
p.add_argument("--customizable", action="store_true", help="emit customize.toml (only when customization was accepted)")
|
||||
p.add_argument("--template", help="override path to the SKILL.md template")
|
||||
p.add_argument("--customize-template", help="override path to the customize.toml template")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
try:
|
||||
result = scaffold(args)
|
||||
except (FileExistsError, FileNotFoundError, ValueError) as e:
|
||||
print(json.dumps({"ok": False, "error": str(e)}))
|
||||
return 1
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# dependencies = ["tiktoken"]
|
||||
# ///
|
||||
"""Deterministic prompt-metrics pre-pass for the Analyze scanners.
|
||||
|
||||
Reads SKILL.md, root-level prompt files, and references, and emits one compact
|
||||
JSON object the LLM scanners read instead of the raw files. Length is reported
|
||||
as tiktoken token counts via count_tokens (cl100k_base, chars//4 fallback);
|
||||
there is no line-count gate anywhere in this script.
|
||||
|
||||
What it surfaces per file:
|
||||
- token count and the counting method (tiktoken or fallback)
|
||||
- frontmatter facts (name, description, description length, angle-bracket flag)
|
||||
- section inventory (heading level + title)
|
||||
- structural signals scanners care about: tables, fenced blocks, defensive
|
||||
padding, meta-explanation, back-references, config header, progression cues
|
||||
|
||||
Budgets the scanners compare against: SKILL.md ~1500-2500 tokens,
|
||||
multi-branch reference ~4500, single-purpose reference ~9000.
|
||||
|
||||
Usage:
|
||||
prepass-prompt-metrics.py <skill-dir> [--output FILE]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Reuse the single length metric rather than reimplementing token counting.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
try:
|
||||
from count_tokens import count_tokens
|
||||
except Exception: # pragma: no cover - count_tokens ships alongside this script
|
||||
def count_tokens(text: str) -> tuple[int, str]:
|
||||
return len(text) // 4, "fallback"
|
||||
|
||||
|
||||
WASTE_PATTERNS = [
|
||||
(r"\b[Mm]ake sure (?:to|you)\b", "defensive-padding", 'Defensive: "make sure to/you"'),
|
||||
(r"\b[Dd]on'?t forget (?:to|that)\b", "defensive-padding", 'Defensive: "don\'t forget"'),
|
||||
(r"\b[Rr]emember (?:to|that)\b", "defensive-padding", 'Defensive: "remember to/that"'),
|
||||
(r"\b[Bb]e sure to\b", "defensive-padding", 'Defensive: "be sure to"'),
|
||||
(r"\b[Pp]lease ensure\b", "defensive-padding", 'Defensive: "please ensure"'),
|
||||
(r"\b[Ii]t is important (?:to|that)\b", "defensive-padding", 'Defensive: "it is important"'),
|
||||
(r"\b[Yy]ou are an AI\b", "meta-explanation", 'Meta: "you are an AI"'),
|
||||
(r"\b[Aa]s a language model\b", "meta-explanation", 'Meta: "as a language model"'),
|
||||
(r"\b[Aa]s an AI assistant\b", "meta-explanation", 'Meta: "as an AI assistant"'),
|
||||
(r"\b[Tt]his (?:workflow|skill|process) is designed to\b", "meta-explanation", 'Meta: "this is designed to"'),
|
||||
(r"\b[Tt]he purpose of this (?:section|step) is\b", "meta-explanation", 'Meta: "the purpose of this is"'),
|
||||
]
|
||||
|
||||
BACKREF_PATTERNS = [
|
||||
(r"\bas described above\b", 'Back-reference: "as described above"'),
|
||||
(r"\bas mentioned (?:above|in|earlier)\b", 'Back-reference: "as mentioned above/earlier"'),
|
||||
(r"\bsee (?:above|the overview)\b", 'Back-reference: "see above/the overview"'),
|
||||
(r"\brefer to (?:the )?(?:above|overview|SKILL)\b", 'Back-reference: "refer to above/overview"'),
|
||||
]
|
||||
|
||||
ALLCAPS_PATTERN = re.compile(r"\b(?:ALWAYS|NEVER|MUST|DO NOT|CRITICAL|REQUIRED)\b")
|
||||
NUMBERED_PREFIX = re.compile(r"^\d{2}[-_]")
|
||||
|
||||
|
||||
def split_frontmatter(content: str) -> tuple[dict, str]:
|
||||
"""Return (frontmatter dict, body). Empty dict when there is no frontmatter."""
|
||||
lines = content.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}, content
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
return {}, content
|
||||
meta: dict[str, str] = {}
|
||||
for line in lines[1:end]:
|
||||
if ":" in line:
|
||||
k, v = line.split(":", 1)
|
||||
meta[k.strip()] = v.strip()
|
||||
return meta, "\n".join(lines[end + 1:])
|
||||
|
||||
|
||||
def count_tables(content: str) -> tuple[int, int]:
|
||||
count = rows = 0
|
||||
in_table = False
|
||||
for line in content.split("\n"):
|
||||
if re.match(r"^\s*\|", line):
|
||||
if not in_table:
|
||||
count += 1
|
||||
in_table = True
|
||||
rows += 1
|
||||
else:
|
||||
in_table = False
|
||||
return count, rows
|
||||
|
||||
|
||||
def count_fenced(content: str) -> int:
|
||||
blocks = 0
|
||||
in_block = False
|
||||
for line in content.split("\n"):
|
||||
if line.strip().startswith("```"):
|
||||
in_block = not in_block
|
||||
if in_block:
|
||||
blocks += 1
|
||||
return blocks
|
||||
|
||||
|
||||
def grep(content: str, lines: list[str], patterns, ignore_case: bool = False) -> list[dict]:
|
||||
flags = re.IGNORECASE if ignore_case else 0
|
||||
hits = []
|
||||
for entry in patterns:
|
||||
pattern, *rest = entry
|
||||
if len(rest) == 2:
|
||||
category, label = rest
|
||||
else:
|
||||
category, label = None, rest[0]
|
||||
for m in re.finditer(pattern, content, flags):
|
||||
ln = content[: m.start()].count("\n") + 1
|
||||
hit = {"line": ln, "pattern": label, "context": lines[ln - 1].strip()[:100]}
|
||||
if category:
|
||||
hit["category"] = category
|
||||
hits.append(hit)
|
||||
return hits
|
||||
|
||||
|
||||
def scan_file(filepath: Path, rel_path: str) -> dict:
|
||||
content = filepath.read_text(encoding="utf-8")
|
||||
lines = content.split("\n")
|
||||
meta, body = split_frontmatter(content)
|
||||
tokens, method = count_tokens(content)
|
||||
|
||||
sections = [
|
||||
{"level": len(m.group(1)), "title": m.group(2).strip()}
|
||||
for m in (re.match(r"^(#{2,4})\s+(.+)$", ln) for ln in lines)
|
||||
if m
|
||||
]
|
||||
|
||||
table_count, table_rows = count_tables(content)
|
||||
allcaps = len(ALLCAPS_PATTERN.findall(content))
|
||||
|
||||
data = {
|
||||
"file": rel_path,
|
||||
"tokens": tokens,
|
||||
"token_method": method,
|
||||
"sections": sections,
|
||||
"table_count": table_count,
|
||||
"table_rows": table_rows,
|
||||
"fenced_block_count": count_fenced(content),
|
||||
"allcaps_directive_count": allcaps,
|
||||
"numbered_prefix_filename": bool(NUMBERED_PREFIX.match(filepath.name)),
|
||||
"waste_patterns": grep(content, lines, WASTE_PATTERNS),
|
||||
"back_references": grep(content, lines, BACKREF_PATTERNS, ignore_case=True),
|
||||
}
|
||||
|
||||
if meta:
|
||||
desc = meta.get("description", "")
|
||||
data["frontmatter"] = {
|
||||
"name": meta.get("name", ""),
|
||||
"description": desc,
|
||||
"description_chars": len(desc),
|
||||
"description_has_angle_brackets": "<" in desc or ">" in desc,
|
||||
"keys": sorted(meta.keys()),
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
def scan(skill_path: Path) -> dict:
|
||||
files_data = []
|
||||
|
||||
skill_md = skill_path / "SKILL.md"
|
||||
if skill_md.exists():
|
||||
d = scan_file(skill_md, "SKILL.md")
|
||||
d["is_skill_md"] = True
|
||||
files_data.append(d)
|
||||
|
||||
for f in sorted(skill_path.iterdir()):
|
||||
if f.is_file() and f.suffix == ".md" and f.name != "SKILL.md":
|
||||
d = scan_file(f, f.name)
|
||||
d["is_skill_md"] = False
|
||||
files_data.append(d)
|
||||
|
||||
references = {}
|
||||
ref_dir = skill_path / "references"
|
||||
if ref_dir.exists():
|
||||
for f in sorted(ref_dir.iterdir()):
|
||||
if f.is_file() and f.suffix in (".md", ".json", ".yaml", ".yml"):
|
||||
tokens, method = count_tokens(f.read_text(encoding="utf-8"))
|
||||
references[f.name] = {
|
||||
"tokens": tokens,
|
||||
"token_method": method,
|
||||
"numbered_prefix_filename": bool(NUMBERED_PREFIX.match(f.name)),
|
||||
}
|
||||
|
||||
skill_md_data = next((f for f in files_data if f.get("is_skill_md")), None)
|
||||
|
||||
return {
|
||||
"scanner": "prompt-metrics-prepass",
|
||||
"script": "prepass-prompt-metrics.py",
|
||||
"version": "2.0.0",
|
||||
"skill_path": str(skill_path),
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"budgets": {
|
||||
"skill_md_tokens": [1500, 2500],
|
||||
"multi_branch_reference_tokens": 4500,
|
||||
"single_purpose_reference_tokens": 9000,
|
||||
},
|
||||
"skill_md": {
|
||||
"tokens": skill_md_data["tokens"] if skill_md_data else 0,
|
||||
"token_method": skill_md_data["token_method"] if skill_md_data else "fallback",
|
||||
"section_count": len(skill_md_data["sections"]) if skill_md_data else 0,
|
||||
"frontmatter": skill_md_data.get("frontmatter") if skill_md_data else None,
|
||||
},
|
||||
"aggregate": {
|
||||
"total_files_scanned": len(files_data),
|
||||
"total_tokens": sum(f["tokens"] for f in files_data),
|
||||
"total_waste_patterns": sum(len(f["waste_patterns"]) for f in files_data),
|
||||
"total_back_references": sum(len(f["back_references"]) for f in files_data),
|
||||
"files_with_numbered_prefix": sum(
|
||||
1 for f in files_data if f["numbered_prefix_filename"]
|
||||
) + sum(1 for r in references.values() if r["numbered_prefix_filename"]),
|
||||
},
|
||||
"reference_sizes": references,
|
||||
"files": files_data,
|
||||
}
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description="Token-based prompt metrics for the Analyze scanners")
|
||||
p.add_argument("skill_path", type=Path, help="path to the skill directory to scan")
|
||||
p.add_argument("--output", "-o", type=Path, help="write JSON to a file instead of stdout")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
if not args.skill_path.is_dir():
|
||||
print(f"error: {args.skill_path} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
output = json.dumps(scan(args.skill_path), indent=2)
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output)
|
||||
print(f"results written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+475
@@ -0,0 +1,475 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic pre-pass for workflow integrity scanner.
|
||||
|
||||
Extracts structural metadata from a BMad skill that the LLM scanner
|
||||
can use instead of reading all files itself. Covers:
|
||||
- Frontmatter parsing and validation
|
||||
- Section inventory (H2/H3 headers)
|
||||
- Template artifact detection
|
||||
- Stage file cross-referencing
|
||||
- Stage numbering validation
|
||||
- Config header detection in prompts
|
||||
- Language/directness pattern grep
|
||||
- On Exit / Exiting section detection (invalid)
|
||||
"""
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Template artifacts that should NOT appear in finalized skills
|
||||
TEMPLATE_ARTIFACTS = [
|
||||
r'\{if-complex-workflow\}', r'\{/if-complex-workflow\}',
|
||||
r'\{if-simple-workflow\}', r'\{/if-simple-workflow\}',
|
||||
r'\{if-simple-utility\}', r'\{/if-simple-utility\}',
|
||||
r'\{if-module\}', r'\{/if-module\}',
|
||||
r'\{if-headless\}', r'\{/if-headless\}',
|
||||
r'\{displayName\}', r'\{skillName\}',
|
||||
]
|
||||
# Runtime variables that ARE expected (not artifacts)
|
||||
RUNTIME_VARS = {
|
||||
'{user_name}', '{communication_language}', '{document_output_language}',
|
||||
'{project-root}', '{output_folder}', '{planning_artifacts}',
|
||||
}
|
||||
|
||||
# Directness anti-patterns
|
||||
DIRECTNESS_PATTERNS = [
|
||||
(r'\byou should\b', 'Suggestive "you should" — use direct imperative'),
|
||||
(r'\bplease\b(?! note)', 'Polite "please" — use direct imperative'),
|
||||
(r'\bhandle appropriately\b', 'Ambiguous "handle appropriately" — specify how'),
|
||||
(r'\bwhen ready\b', 'Vague "when ready" — specify testable condition'),
|
||||
]
|
||||
|
||||
# Invalid sections
|
||||
INVALID_SECTIONS = [
|
||||
(r'^##\s+On\s+Exit\b', 'On Exit section found — no exit hooks exist in the system, this will never run'),
|
||||
(r'^##\s+Exiting\b', 'Exiting section found — no exit hooks exist in the system, this will never run'),
|
||||
]
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> tuple[dict | None, list[dict]]:
|
||||
"""Parse YAML frontmatter and validate."""
|
||||
findings = []
|
||||
fm_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
|
||||
if not fm_match:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'critical', 'category': 'frontmatter',
|
||||
'issue': 'No YAML frontmatter found',
|
||||
})
|
||||
return None, findings
|
||||
|
||||
try:
|
||||
# Frontmatter is YAML-like key: value pairs — parse manually
|
||||
fm = {}
|
||||
for line in fm_match.group(1).strip().split('\n'):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if ':' in line:
|
||||
key, _, value = line.partition(':')
|
||||
fm[key.strip()] = value.strip().strip('"').strip("'")
|
||||
except Exception as e:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'critical', 'category': 'frontmatter',
|
||||
'issue': f'Invalid frontmatter: {e}',
|
||||
})
|
||||
return None, findings
|
||||
|
||||
if not isinstance(fm, dict):
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'critical', 'category': 'frontmatter',
|
||||
'issue': 'Frontmatter is not a YAML mapping',
|
||||
})
|
||||
return None, findings
|
||||
|
||||
# name check
|
||||
name = fm.get('name')
|
||||
if not name:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'critical', 'category': 'frontmatter',
|
||||
'issue': 'Missing "name" field in frontmatter',
|
||||
})
|
||||
elif not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name):
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'high', 'category': 'frontmatter',
|
||||
'issue': f'Name "{name}" is not kebab-case',
|
||||
})
|
||||
# bmad- prefix check removed — bmad- is reserved for official BMad creations only
|
||||
|
||||
# description check
|
||||
desc = fm.get('description')
|
||||
if not desc:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'high', 'category': 'frontmatter',
|
||||
'issue': 'Missing "description" field in frontmatter',
|
||||
})
|
||||
elif 'Use when' not in desc and 'use when' not in desc:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'medium', 'category': 'frontmatter',
|
||||
'issue': 'Description missing "Use when..." trigger phrase',
|
||||
})
|
||||
|
||||
# Extra fields check
|
||||
allowed = {'name', 'description', 'menu-code'}
|
||||
extra = set(fm.keys()) - allowed
|
||||
if extra:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'low', 'category': 'frontmatter',
|
||||
'issue': f'Extra frontmatter fields: {", ".join(sorted(extra))}',
|
||||
})
|
||||
|
||||
return fm, findings
|
||||
|
||||
|
||||
def extract_sections(content: str) -> list[dict]:
|
||||
"""Extract all H2 headers with line numbers."""
|
||||
sections = []
|
||||
for i, line in enumerate(content.split('\n'), 1):
|
||||
m = re.match(r'^(#{2,3})\s+(.+)$', line)
|
||||
if m:
|
||||
sections.append({
|
||||
'level': len(m.group(1)),
|
||||
'title': m.group(2).strip(),
|
||||
'line': i,
|
||||
})
|
||||
return sections
|
||||
|
||||
|
||||
def check_required_sections(sections: list[dict]) -> list[dict]:
|
||||
"""Check for required and invalid sections."""
|
||||
findings = []
|
||||
h2_titles = [s['title'] for s in sections if s['level'] == 2]
|
||||
|
||||
if 'Overview' not in h2_titles:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'high', 'category': 'sections',
|
||||
'issue': 'Missing ## Overview section',
|
||||
})
|
||||
|
||||
if 'On Activation' not in h2_titles:
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 1,
|
||||
'severity': 'high', 'category': 'sections',
|
||||
'issue': 'Missing ## On Activation section',
|
||||
})
|
||||
|
||||
# Invalid sections
|
||||
for s in sections:
|
||||
if s['level'] == 2:
|
||||
for pattern, message in INVALID_SECTIONS:
|
||||
if re.match(pattern, f"## {s['title']}"):
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': s['line'],
|
||||
'severity': 'high', 'category': 'invalid-section',
|
||||
'issue': message,
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def find_template_artifacts(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Scan for orphaned template substitution artifacts."""
|
||||
findings = []
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
|
||||
for pattern in TEMPLATE_ARTIFACTS:
|
||||
for m in re.finditer(pattern, content):
|
||||
matched = m.group()
|
||||
if matched in RUNTIME_VARS:
|
||||
continue
|
||||
line_num = content[:m.start()].count('\n') + 1
|
||||
findings.append({
|
||||
'file': rel_path, 'line': line_num,
|
||||
'severity': 'high', 'category': 'artifacts',
|
||||
'issue': f'Orphaned template artifact: {matched}',
|
||||
'fix': 'Resolve or remove this template conditional/placeholder',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def cross_reference_stages(skill_path: Path, skill_content: str) -> tuple[dict, list[dict]]:
|
||||
"""Cross-reference stage files between SKILL.md and numbered prompt files at skill root."""
|
||||
findings = []
|
||||
|
||||
# Get actual numbered prompt files at skill root (exclude SKILL.md)
|
||||
actual_files = set()
|
||||
for f in skill_path.iterdir():
|
||||
if f.is_file() and f.suffix == '.md' and f.name != 'SKILL.md' and re.match(r'^\d+-', f.name):
|
||||
actual_files.add(f.name)
|
||||
|
||||
# Find stage references in SKILL.md — look for both old prompts/ style and new root style
|
||||
referenced = set()
|
||||
# Match `prompts/XX-name.md` (legacy) or bare `XX-name.md` references
|
||||
ref_pattern = re.compile(r'(?:prompts/)?(\d+-[^\s)`]+\.md)')
|
||||
for m in ref_pattern.finditer(skill_content):
|
||||
referenced.add(m.group(1))
|
||||
|
||||
# Missing files (referenced but don't exist)
|
||||
missing = referenced - actual_files
|
||||
for f in sorted(missing):
|
||||
findings.append({
|
||||
'file': 'SKILL.md', 'line': 0,
|
||||
'severity': 'critical', 'category': 'missing-stage',
|
||||
'issue': f'Referenced stage file does not exist: {f}',
|
||||
})
|
||||
|
||||
# Orphaned files (exist but not referenced)
|
||||
orphaned = actual_files - referenced
|
||||
for f in sorted(orphaned):
|
||||
findings.append({
|
||||
'file': f, 'line': 0,
|
||||
'severity': 'medium', 'category': 'naming',
|
||||
'issue': f'Stage file exists but not referenced in SKILL.md: {f}',
|
||||
})
|
||||
|
||||
# Stage numbering check
|
||||
numbered = []
|
||||
for f in sorted(actual_files):
|
||||
m = re.match(r'^(\d+)-(.+)\.md$', f)
|
||||
if m:
|
||||
numbered.append((int(m.group(1)), f))
|
||||
|
||||
if numbered:
|
||||
numbered.sort()
|
||||
nums = [n[0] for n in numbered]
|
||||
expected = list(range(nums[0], nums[0] + len(nums)))
|
||||
if nums != expected:
|
||||
gaps = set(expected) - set(nums)
|
||||
if gaps:
|
||||
findings.append({
|
||||
'file': skill_path.name, 'line': 0,
|
||||
'severity': 'medium', 'category': 'naming',
|
||||
'issue': f'Stage numbering has gaps: missing {sorted(gaps)}',
|
||||
})
|
||||
|
||||
stage_summary = {
|
||||
'total_stages': len(actual_files),
|
||||
'referenced': sorted(referenced),
|
||||
'actual': sorted(actual_files),
|
||||
'missing_stages': sorted(missing),
|
||||
'orphaned_stages': sorted(orphaned),
|
||||
}
|
||||
|
||||
return stage_summary, findings
|
||||
|
||||
|
||||
def check_prompt_basics(skill_path: Path) -> tuple[list[dict], list[dict]]:
|
||||
"""Check each prompt file for config header and progression conditions."""
|
||||
findings = []
|
||||
prompt_details = []
|
||||
|
||||
# Look for numbered prompt files at skill root
|
||||
prompt_files = sorted(
|
||||
f for f in skill_path.iterdir()
|
||||
if f.is_file() and f.suffix == '.md' and f.name != 'SKILL.md' and re.match(r'^\d+-', f.name)
|
||||
)
|
||||
if not prompt_files:
|
||||
return prompt_details, findings
|
||||
|
||||
for f in prompt_files:
|
||||
content = f.read_text(encoding='utf-8')
|
||||
rel_path = f.name
|
||||
detail = {'file': f.name, 'has_config_header': False, 'has_progression': False}
|
||||
|
||||
# Config header check
|
||||
if '{communication_language}' in content or '{document_output_language}' in content:
|
||||
detail['has_config_header'] = True
|
||||
else:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'config-header',
|
||||
'issue': 'No config header with language variables found',
|
||||
})
|
||||
|
||||
# Progression condition check (look for progression-related keywords near end)
|
||||
lower = content.lower()
|
||||
prog_keywords = ['progress', 'advance', 'move to', 'next stage', 'when complete',
|
||||
'proceed to', 'transition', 'completion criteria']
|
||||
if any(kw in lower for kw in prog_keywords):
|
||||
detail['has_progression'] = True
|
||||
else:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': len(content.split('\n')),
|
||||
'severity': 'high', 'category': 'progression',
|
||||
'issue': 'No progression condition keywords found',
|
||||
})
|
||||
|
||||
# Directness checks
|
||||
for pattern, message in DIRECTNESS_PATTERNS:
|
||||
for m in re.finditer(pattern, content, re.IGNORECASE):
|
||||
line_num = content[:m.start()].count('\n') + 1
|
||||
findings.append({
|
||||
'file': rel_path, 'line': line_num,
|
||||
'severity': 'low', 'category': 'language',
|
||||
'issue': message,
|
||||
})
|
||||
|
||||
# Template artifacts
|
||||
findings.extend(find_template_artifacts(f, rel_path))
|
||||
|
||||
prompt_details.append(detail)
|
||||
|
||||
return prompt_details, findings
|
||||
|
||||
|
||||
def detect_workflow_type(skill_content: str, has_prompts: bool) -> str:
|
||||
"""Detect workflow type from SKILL.md content."""
|
||||
has_stage_refs = bool(re.search(r'(?:prompts/)?\d+-\S+\.md', skill_content))
|
||||
has_routing = bool(re.search(r'(?i)(rout|stage|branch|path)', skill_content))
|
||||
|
||||
if has_stage_refs or (has_prompts and has_routing):
|
||||
return 'complex'
|
||||
elif re.search(r'(?m)^\d+\.\s', skill_content):
|
||||
return 'simple-workflow'
|
||||
else:
|
||||
return 'simple-utility'
|
||||
|
||||
|
||||
def scan_workflow_integrity(skill_path: Path) -> dict:
|
||||
"""Run all deterministic workflow integrity checks."""
|
||||
all_findings = []
|
||||
|
||||
# Read SKILL.md
|
||||
skill_md = skill_path / 'SKILL.md'
|
||||
if not skill_md.exists():
|
||||
return {
|
||||
'scanner': 'workflow-integrity-prepass',
|
||||
'script': 'prepass-workflow-integrity.py',
|
||||
'version': '1.0.0',
|
||||
'skill_path': str(skill_path),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'status': 'fail',
|
||||
'issues': [{'file': 'SKILL.md', 'line': 1, 'severity': 'critical',
|
||||
'category': 'missing-file', 'issue': 'SKILL.md does not exist'}],
|
||||
'summary': {'total_issues': 1, 'by_severity': {'critical': 1, 'high': 0, 'medium': 0, 'low': 0}},
|
||||
}
|
||||
|
||||
skill_content = skill_md.read_text(encoding='utf-8')
|
||||
|
||||
# Frontmatter
|
||||
frontmatter, fm_findings = parse_frontmatter(skill_content)
|
||||
all_findings.extend(fm_findings)
|
||||
|
||||
# Sections
|
||||
sections = extract_sections(skill_content)
|
||||
section_findings = check_required_sections(sections)
|
||||
all_findings.extend(section_findings)
|
||||
|
||||
# Template artifacts in SKILL.md
|
||||
all_findings.extend(find_template_artifacts(skill_md, 'SKILL.md'))
|
||||
|
||||
# Directness checks in SKILL.md
|
||||
for pattern, message in DIRECTNESS_PATTERNS:
|
||||
for m in re.finditer(pattern, skill_content, re.IGNORECASE):
|
||||
line_num = skill_content[:m.start()].count('\n') + 1
|
||||
all_findings.append({
|
||||
'file': 'SKILL.md', 'line': line_num,
|
||||
'severity': 'low', 'category': 'language',
|
||||
'issue': message,
|
||||
})
|
||||
|
||||
# Workflow type
|
||||
has_prompts = any(
|
||||
f.is_file() and f.suffix == '.md' and f.name != 'SKILL.md' and re.match(r'^\d+-', f.name)
|
||||
for f in skill_path.iterdir()
|
||||
)
|
||||
workflow_type = detect_workflow_type(skill_content, has_prompts)
|
||||
|
||||
# Stage cross-reference
|
||||
stage_summary, stage_findings = cross_reference_stages(skill_path, skill_content)
|
||||
all_findings.extend(stage_findings)
|
||||
|
||||
# Prompt basics
|
||||
prompt_details, prompt_findings = check_prompt_basics(skill_path)
|
||||
all_findings.extend(prompt_findings)
|
||||
|
||||
# Build severity summary
|
||||
by_severity = {'critical': 0, 'high': 0, 'medium': 0, 'low': 0}
|
||||
for f in all_findings:
|
||||
sev = f['severity']
|
||||
if sev in by_severity:
|
||||
by_severity[sev] += 1
|
||||
|
||||
status = 'pass'
|
||||
if by_severity['critical'] > 0:
|
||||
status = 'fail'
|
||||
elif by_severity['high'] > 0:
|
||||
status = 'warning'
|
||||
|
||||
return {
|
||||
'scanner': 'workflow-integrity-prepass',
|
||||
'script': 'prepass-workflow-integrity.py',
|
||||
'version': '1.0.0',
|
||||
'skill_path': str(skill_path),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'status': status,
|
||||
'metadata': {
|
||||
'frontmatter': frontmatter,
|
||||
'sections': sections,
|
||||
'workflow_type': workflow_type,
|
||||
},
|
||||
'stage_summary': stage_summary,
|
||||
'prompt_details': prompt_details,
|
||||
'issues': all_findings,
|
||||
'summary': {
|
||||
'total_issues': len(all_findings),
|
||||
'by_severity': by_severity,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Deterministic pre-pass for workflow integrity scanning',
|
||||
)
|
||||
parser.add_argument(
|
||||
'skill_path',
|
||||
type=Path,
|
||||
help='Path to the skill directory to scan',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output', '-o',
|
||||
type=Path,
|
||||
help='Write JSON output to file instead of stdout',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.skill_path.is_dir():
|
||||
print(f"Error: {args.skill_path} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
result = scan_workflow_integrity(args.skill_path)
|
||||
output = json.dumps(result, indent=2)
|
||||
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output)
|
||||
print(f"Results written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0 if result['status'] == 'pass' else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# ///
|
||||
"""quick_validate — structural lint for a skill's SKILL.md frontmatter.
|
||||
|
||||
Checks the few things a structural error makes obvious: the frontmatter parses,
|
||||
it carries only allowed keys, the name is hyphen-case and within length, and the
|
||||
description is present, within bounds, and free of angle brackets (which break
|
||||
the router). The allowed-key set is configurable, never baked to one provider:
|
||||
pass --allow-key to extend it or --allow-keys to replace it.
|
||||
|
||||
Exit code is 0 when every check passes and 1 when any check fails, so a build or
|
||||
CI step can gate on it. Findings print as one JSON object on stdout.
|
||||
|
||||
Usage:
|
||||
quick_validate.py <skill-dir-or-SKILL.md>
|
||||
quick_validate.py <path> --allow-key license --allow-key version
|
||||
quick_validate.py <path> --allow-keys name,description
|
||||
quick_validate.py <path> --max-name 64 --max-desc 1024
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_ALLOWED_KEYS = ["name", "description"]
|
||||
HYPHEN_CASE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
|
||||
|
||||
def split_frontmatter(content: str):
|
||||
"""Return (ok, frontmatter dict, error). ok is False when there is no parseable block."""
|
||||
lines = content.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return False, {}, "no frontmatter block (file does not open with ---)"
|
||||
end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None)
|
||||
if end is None:
|
||||
return False, {}, "frontmatter block is not terminated with a closing ---"
|
||||
meta: dict[str, str] = {}
|
||||
for line in lines[1:end]:
|
||||
if not line.strip():
|
||||
continue
|
||||
if ":" not in line:
|
||||
return False, {}, f"frontmatter line is not key: value -> {line.strip()!r}"
|
||||
k, v = line.split(":", 1)
|
||||
meta[k.strip()] = v.strip()
|
||||
return True, meta, ""
|
||||
|
||||
|
||||
def validate(content: str, allowed_keys, max_name: int, max_desc: int) -> list[dict]:
|
||||
errors: list[dict] = []
|
||||
|
||||
ok, meta, parse_error = split_frontmatter(content)
|
||||
if not ok:
|
||||
return [{"check": "frontmatter", "message": parse_error}]
|
||||
|
||||
extra = [k for k in meta if k not in allowed_keys]
|
||||
if extra:
|
||||
errors.append({
|
||||
"check": "allowed-keys",
|
||||
"message": f"unexpected frontmatter keys: {', '.join(sorted(extra))}; allowed: {', '.join(allowed_keys)}",
|
||||
})
|
||||
|
||||
name = meta.get("name", "")
|
||||
if not name:
|
||||
errors.append({"check": "name", "message": "name is missing or empty"})
|
||||
else:
|
||||
if not HYPHEN_CASE.match(name):
|
||||
errors.append({"check": "name", "message": f"name {name!r} is not hyphen-case (lowercase, digits, single hyphens)"})
|
||||
if len(name) > max_name:
|
||||
errors.append({"check": "name", "message": f"name is {len(name)} chars, over the {max_name} limit"})
|
||||
|
||||
desc = meta.get("description", "")
|
||||
if not desc:
|
||||
errors.append({"check": "description", "message": "description is missing or empty"})
|
||||
else:
|
||||
if len(desc) > max_desc:
|
||||
errors.append({"check": "description", "message": f"description is {len(desc)} chars, over the {max_desc} limit"})
|
||||
if "<" in desc or ">" in desc:
|
||||
errors.append({"check": "description", "message": "description contains angle brackets, which break router matching"})
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def resolve_skill_md(path: Path) -> Path:
|
||||
return path / "SKILL.md" if path.is_dir() else path
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(description="Structural lint for a skill's SKILL.md frontmatter")
|
||||
p.add_argument("path", type=Path, help="skill directory or a SKILL.md file")
|
||||
p.add_argument("--allow-key", action="append", default=[], help="add one key to the allowed set (repeatable)")
|
||||
p.add_argument("--allow-keys", help="comma-separated set that REPLACES the default allowed keys")
|
||||
p.add_argument("--max-name", type=int, default=64, help="max name length (default 64)")
|
||||
p.add_argument("--max-desc", type=int, default=1024, help="max description length (default 1024)")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
skill_md = resolve_skill_md(args.path)
|
||||
if not skill_md.is_file():
|
||||
print(json.dumps({"ok": False, "errors": [{"check": "path", "message": f"{skill_md} not found"}]}))
|
||||
return 1
|
||||
|
||||
if args.allow_keys:
|
||||
allowed = [k.strip() for k in args.allow_keys.split(",") if k.strip()]
|
||||
else:
|
||||
allowed = list(DEFAULT_ALLOWED_KEYS) + list(args.allow_key)
|
||||
|
||||
errors = validate(skill_md.read_text(encoding="utf-8"), allowed, args.max_name, args.max_desc)
|
||||
print(json.dumps({"ok": not errors, "file": str(skill_md), "errors": errors}))
|
||||
return 0 if not errors else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# ///
|
||||
"""Render the analysis report deterministically from findings JSON.
|
||||
|
||||
Injects a validated findings JSON object into the report shell's
|
||||
report-data island and writes the self-contained HTML atomically.
|
||||
With --md, also writes a markdown rendering of the same data as the
|
||||
archival artifact.
|
||||
|
||||
Refuses (non-zero exit, message on stderr) when the JSON does not
|
||||
parse, fails shape validation, or still carries the shell's
|
||||
placeholder subject — a refused render means fix the findings file
|
||||
and re-run, never hand-edit the HTML.
|
||||
|
||||
Usage:
|
||||
uv run render_report.py <findings.json> --shell <report-shell.html> \
|
||||
-o <out.html> [--md <out.md>]
|
||||
|
||||
On success prints one JSON line: output paths, grade, and severity
|
||||
counts derived from the findings array.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SEVERITIES = ("critical", "high", "medium", "low")
|
||||
GRADES = ("excellent", "good", "fair", "poor")
|
||||
PLACEHOLDER_SUBJECT = "__PLACEHOLDER__"
|
||||
ISLAND_RE = re.compile(
|
||||
r'(<script[^>]*\bid="report-data"[^>]*>)(.*?)(</script>)', re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
print(f"render_report: {message}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def validate(data: object) -> list[str]:
|
||||
"""Return a list of shape errors; empty list means valid."""
|
||||
if not isinstance(data, dict):
|
||||
return ["top level must be a JSON object"]
|
||||
errors: list[str] = []
|
||||
|
||||
subject = data.get("subject")
|
||||
if not isinstance(subject, str) or not subject.strip():
|
||||
errors.append('"subject" must be a non-empty string')
|
||||
elif PLACEHOLDER_SUBJECT in subject:
|
||||
errors.append(
|
||||
f'"subject" still carries the placeholder {PLACEHOLDER_SUBJECT}; '
|
||||
"this is the unfilled shell sample, not real findings"
|
||||
)
|
||||
|
||||
findings = data.get("findings")
|
||||
if not isinstance(findings, list):
|
||||
errors.append('"findings" must be an array (use [] for a clean pass)')
|
||||
else:
|
||||
for i, finding in enumerate(findings):
|
||||
if not isinstance(finding, dict):
|
||||
errors.append(f"findings[{i}] must be an object")
|
||||
|
||||
grade = data.get("grade")
|
||||
if grade is not None and grade not in GRADES:
|
||||
errors.append(f'"grade" must be one of: {", ".join(GRADES)}')
|
||||
|
||||
for key in ("themes", "recommendations"):
|
||||
value = data.get(key)
|
||||
if value is not None and (
|
||||
not isinstance(value, list)
|
||||
or any(not isinstance(item, dict) for item in value)
|
||||
):
|
||||
errors.append(f'"{key}" must be an array of objects')
|
||||
|
||||
strengths = data.get("strengths")
|
||||
if strengths is not None and (
|
||||
not isinstance(strengths, list)
|
||||
or any(not isinstance(item, str) for item in strengths)
|
||||
):
|
||||
errors.append('"strengths" must be an array of strings')
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def severity_counts(findings: list[dict]) -> dict[str, int]:
|
||||
counts = {sev: 0 for sev in SEVERITIES}
|
||||
for finding in findings:
|
||||
sev = finding.get("severity")
|
||||
counts[sev if sev in counts else "low"] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def inject(shell_html: str, data: dict) -> str:
|
||||
payload = json.dumps(data, ensure_ascii=False, indent=2)
|
||||
# A "</" sequence inside a JSON string would close the script tag
|
||||
# early in the browser; "<\/" is the same string to JSON.parse.
|
||||
payload = payload.replace("</", "<\\/")
|
||||
|
||||
def replace(match: re.Match) -> str:
|
||||
return match.group(1) + "\n" + payload + "\n" + match.group(3)
|
||||
|
||||
new_html, count = ISLAND_RE.subn(replace, shell_html, count=1)
|
||||
if count != 1:
|
||||
fail('shell has no <script id="report-data"> island to fill')
|
||||
return new_html
|
||||
|
||||
|
||||
def atomic_write(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(
|
||||
dir=path.parent, prefix=path.name + ".", suffix=".tmp"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
handle.write(text)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _finding_lines(finding: dict, heading_level: str) -> list[str]:
|
||||
fid = str(finding.get("id", ""))
|
||||
title = str(finding.get("title", "(untitled finding)"))
|
||||
lines = [f"{heading_level} {fid} — {title}" if fid else f"{heading_level} {title}", ""]
|
||||
for key, label in (
|
||||
("lens", "Lens"),
|
||||
("location", "Location"),
|
||||
("evidence", "Evidence"),
|
||||
("recommendation", "Recommendation"),
|
||||
("proposed_smallest", "Proposed smallest"),
|
||||
("predicted_delta", "Predicted delta"),
|
||||
):
|
||||
value = finding.get(key)
|
||||
if value:
|
||||
value = f"`{value}`" if key == "location" else str(value)
|
||||
lines.append(f"- {label}: {value}")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def render_md(data: dict) -> str:
|
||||
findings = [f for f in data.get("findings", []) if isinstance(f, dict)]
|
||||
by_id = {str(f.get("id")): f for f in findings if f.get("id") is not None}
|
||||
counts = severity_counts(findings)
|
||||
lines: list[str] = []
|
||||
|
||||
lines.append(f"# Analysis Report: {data.get('subject', '')}")
|
||||
lines.append("")
|
||||
meta = []
|
||||
if data.get("generated"):
|
||||
meta.append(f"Generated: {data['generated']}")
|
||||
if data.get("schema_version") is not None:
|
||||
meta.append(f"Schema: {data['schema_version']}")
|
||||
if meta:
|
||||
lines.append(" · ".join(meta))
|
||||
lines.append("")
|
||||
|
||||
if data.get("grade"):
|
||||
lines.append(f"**Grade: {str(data['grade']).capitalize()}**")
|
||||
lines.append("")
|
||||
if data.get("verdict"):
|
||||
lines.append(f"> {data['verdict']}")
|
||||
lines.append("")
|
||||
summary = data.get("summary")
|
||||
if isinstance(summary, str) and summary:
|
||||
lines.append(summary)
|
||||
lines.append("")
|
||||
|
||||
lines.append("| Severity | Count |")
|
||||
lines.append("| --- | --- |")
|
||||
for sev in SEVERITIES:
|
||||
lines.append(f"| {sev.capitalize()} | {counts[sev]} |")
|
||||
lines.append("")
|
||||
|
||||
themes = data.get("themes") or []
|
||||
if themes:
|
||||
lines.append("## Themes")
|
||||
lines.append("")
|
||||
for i, theme in enumerate(themes, 1):
|
||||
lines.append(f"### {i}. {theme.get('title', '(untitled theme)')}")
|
||||
lines.append("")
|
||||
if theme.get("root_cause"):
|
||||
lines.append(f"- Root cause: {theme['root_cause']}")
|
||||
if theme.get("action"):
|
||||
lines.append(f"- Fix: {theme['action']}")
|
||||
ids = theme.get("finding_ids") or []
|
||||
if ids:
|
||||
lines.append("- Findings:")
|
||||
for fid in ids:
|
||||
finding = by_id.get(str(fid))
|
||||
if finding:
|
||||
loc = finding.get("location")
|
||||
suffix = f" — `{loc}`" if loc else ""
|
||||
lines.append(
|
||||
f" - `{fid}` {finding.get('title', '')}{suffix}"
|
||||
)
|
||||
else:
|
||||
lines.append(f" - `{fid}`")
|
||||
lines.append("")
|
||||
|
||||
strengths = data.get("strengths") or []
|
||||
if strengths:
|
||||
lines.append("## Strengths")
|
||||
lines.append("")
|
||||
for strength in strengths:
|
||||
lines.append(f"- {strength}")
|
||||
lines.append("")
|
||||
|
||||
recommendations = data.get("recommendations") or []
|
||||
if recommendations:
|
||||
lines.append("## Recommendations")
|
||||
lines.append("")
|
||||
for i, rec in enumerate(recommendations, 1):
|
||||
rank = rec.get("rank", i)
|
||||
resolves = rec.get("resolves")
|
||||
if isinstance(resolves, list) and resolves:
|
||||
suffix = " (resolves: " + ", ".join(map(str, resolves)) + ")"
|
||||
elif isinstance(resolves, (int, float)):
|
||||
suffix = f" (resolves {int(resolves)} findings)"
|
||||
else:
|
||||
suffix = ""
|
||||
lines.append(f"{rank}. {rec.get('action', '')}{suffix}")
|
||||
lines.append("")
|
||||
|
||||
# Optional agent blocks: rendered only when present so the same
|
||||
# renderer serves both the workflow and agent schemas.
|
||||
profile = data.get("agent_profile")
|
||||
if isinstance(profile, dict) and any(profile.values()):
|
||||
lines.append("## Agent Profile")
|
||||
lines.append("")
|
||||
for key, label in (
|
||||
("name", "Name"),
|
||||
("title", "Title"),
|
||||
("agent_type", "Type"),
|
||||
("mission", "Mission"),
|
||||
):
|
||||
if profile.get(key):
|
||||
lines.append(f"- {label}: {profile[key]}")
|
||||
lines.append("")
|
||||
|
||||
capabilities = data.get("capabilities")
|
||||
if isinstance(capabilities, list) and capabilities:
|
||||
lines.append("## Capabilities")
|
||||
lines.append("")
|
||||
for cap in capabilities:
|
||||
if not isinstance(cap, dict) or not cap.get("name"):
|
||||
continue
|
||||
kind = f" ({cap['kind']})" if cap.get("kind") else ""
|
||||
note = f" — {cap['note']}" if cap.get("note") else ""
|
||||
lines.append(f"- **{cap['name']}**{kind}{note}")
|
||||
lines.append("")
|
||||
|
||||
detailed = data.get("detailed_analysis")
|
||||
if isinstance(detailed, dict) and detailed:
|
||||
lines.append("## Per-Lens Verdicts")
|
||||
lines.append("")
|
||||
for lens, verdict in detailed.items():
|
||||
if verdict:
|
||||
lines.append(f"- **{lens}**: {verdict}")
|
||||
lines.append("")
|
||||
|
||||
sanctum = data.get("sanctum")
|
||||
if isinstance(sanctum, dict) and sanctum.get("present") is not False:
|
||||
rows = []
|
||||
if sanctum.get("location"):
|
||||
rows.append(f"- Location: `{sanctum['location']}`")
|
||||
files = sanctum.get("files") or []
|
||||
if files:
|
||||
rows.append("- Files: " + ", ".join(f"`{f}`" for f in files))
|
||||
if sanctum.get("note"):
|
||||
rows.append(f"- Note: {sanctum['note']}")
|
||||
if rows:
|
||||
lines.append("## Sanctum (runtime memory)")
|
||||
lines.append("")
|
||||
lines.extend(rows)
|
||||
lines.append("")
|
||||
|
||||
experience = data.get("experience")
|
||||
if isinstance(experience, dict):
|
||||
journeys = [
|
||||
j for j in experience.get("journeys") or [] if isinstance(j, dict)
|
||||
]
|
||||
headless = experience.get("headless")
|
||||
if journeys or headless:
|
||||
lines.append("## Experience")
|
||||
lines.append("")
|
||||
for journey in journeys:
|
||||
steps = f" — {journey['steps']}" if journey.get("steps") else ""
|
||||
lines.append(f"- **{journey.get('name', '(unnamed journey)')}**{steps}")
|
||||
if headless:
|
||||
lines.append(f"- Headless: {headless}")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Findings")
|
||||
lines.append("")
|
||||
if not findings:
|
||||
lines.append("No findings: the scanners returned a clean pass.")
|
||||
lines.append("")
|
||||
else:
|
||||
for sev in SEVERITIES:
|
||||
group = [
|
||||
f
|
||||
for f in findings
|
||||
if (f.get("severity") if f.get("severity") in SEVERITIES else "low")
|
||||
== sev
|
||||
]
|
||||
if not group:
|
||||
continue
|
||||
lines.append(f"### {sev.capitalize()} ({len(group)})")
|
||||
lines.append("")
|
||||
for finding in group:
|
||||
lines.extend(_finding_lines(finding, "####"))
|
||||
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Inject findings JSON into the report shell and render HTML (+ optional markdown)."
|
||||
)
|
||||
parser.add_argument("findings", type=Path, help="path to findings.json")
|
||||
parser.add_argument(
|
||||
"--shell", type=Path, required=True, help="path to report-shell.html"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o", "--output", type=Path, required=True, help="output HTML path"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--md", type=Path, help="also write a markdown rendering to this path"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
raw = args.findings.read_text(encoding="utf-8")
|
||||
except OSError as err:
|
||||
fail(f"cannot read {args.findings}: {err}")
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as err:
|
||||
fail(f"{args.findings} is not valid JSON: {err}")
|
||||
|
||||
errors = validate(data)
|
||||
if errors:
|
||||
fail(
|
||||
f"{args.findings} failed shape validation:\n - "
|
||||
+ "\n - ".join(errors)
|
||||
)
|
||||
|
||||
try:
|
||||
shell_html = args.shell.read_text(encoding="utf-8")
|
||||
except OSError as err:
|
||||
fail(f"cannot read shell {args.shell}: {err}")
|
||||
|
||||
atomic_write(args.output, inject(shell_html, data))
|
||||
if args.md:
|
||||
atomic_write(args.md, render_md(data))
|
||||
|
||||
findings = [f for f in data.get("findings", []) if isinstance(f, dict)]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"html_report": str(args.output),
|
||||
"md_report": str(args.md) if args.md else None,
|
||||
"grade": data.get("grade"),
|
||||
"counts": severity_counts(findings),
|
||||
"findings": len(findings),
|
||||
}
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic path standards scanner for BMad skills.
|
||||
|
||||
Validates all .md and .json files against BMad path conventions:
|
||||
1. {project-root} for any project-scope path (not just _bmad)
|
||||
2. Bare _bmad references must have {project-root} prefix
|
||||
3. Config variables used directly — no double-prefix with {project-root}
|
||||
4. ./ only for same-folder references — never ./subdir/ cross-directory
|
||||
5. No ../ parent directory references
|
||||
6. No absolute paths
|
||||
7. Frontmatter allows only name and description
|
||||
8. No .md files at skill root except SKILL.md
|
||||
"""
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Patterns to detect
|
||||
# Double-prefix: {project-root}/{config-variable} — config vars already contain project-root
|
||||
DOUBLE_PREFIX_RE = re.compile(r'\{project-root\}/\{[^}]+\}')
|
||||
# Bare _bmad without {project-root} prefix — match _bmad at word boundary
|
||||
# but not when preceded by {project-root}/
|
||||
BARE_BMAD_RE = re.compile(r'(?<!\{project-root\}/)_bmad[/\s]')
|
||||
# Absolute paths
|
||||
ABSOLUTE_PATH_RE = re.compile(r'(?:^|[\s"`\'(])(/(?:Users|home|opt|var|tmp|etc|usr)/\S+)', re.MULTILINE)
|
||||
HOME_PATH_RE = re.compile(r'(?:^|[\s"`\'(])(~/\S+)', re.MULTILINE)
|
||||
# Parent directory reference (still invalid)
|
||||
RELATIVE_DOT_RE = re.compile(r'(?:^|[\s"`\'(])(\.\./\S+)', re.MULTILINE)
|
||||
# Cross-directory ./ — ./subdir/ is wrong because ./ means same folder only
|
||||
CROSS_DIR_DOT_SLASH_RE = re.compile(r'(?:^|[\s"`\'(])\./(?:references|scripts|assets)/\S+', re.MULTILINE)
|
||||
|
||||
# Fenced code block detection (to skip examples showing wrong patterns)
|
||||
FENCE_RE = re.compile(r'^```', re.MULTILINE)
|
||||
|
||||
# Valid frontmatter keys
|
||||
VALID_FRONTMATTER_KEYS = {'name', 'description'}
|
||||
|
||||
|
||||
def is_in_fenced_block(content: str, pos: int) -> bool:
|
||||
"""Check if a position is inside a fenced code block."""
|
||||
fences = [m.start() for m in FENCE_RE.finditer(content[:pos])]
|
||||
# Odd number of fences before pos means we're inside a block
|
||||
return len(fences) % 2 == 1
|
||||
|
||||
|
||||
def get_line_number(content: str, pos: int) -> int:
|
||||
"""Get 1-based line number for a position in content."""
|
||||
return content[:pos].count('\n') + 1
|
||||
|
||||
|
||||
def check_frontmatter(content: str, filepath: Path) -> list[dict]:
|
||||
"""Validate SKILL.md frontmatter contains only allowed keys."""
|
||||
findings = []
|
||||
if filepath.name != 'SKILL.md':
|
||||
return findings
|
||||
|
||||
if not content.startswith('---'):
|
||||
findings.append({
|
||||
'file': filepath.name,
|
||||
'line': 1,
|
||||
'severity': 'critical',
|
||||
'category': 'frontmatter',
|
||||
'title': 'SKILL.md missing frontmatter block',
|
||||
'detail': 'SKILL.md must start with --- frontmatter containing name and description',
|
||||
'action': 'Add frontmatter with name and description fields',
|
||||
})
|
||||
return findings
|
||||
|
||||
# Find closing ---
|
||||
end = content.find('\n---', 3)
|
||||
if end == -1:
|
||||
findings.append({
|
||||
'file': filepath.name,
|
||||
'line': 1,
|
||||
'severity': 'critical',
|
||||
'category': 'frontmatter',
|
||||
'title': 'SKILL.md frontmatter block not closed',
|
||||
'detail': 'Missing closing --- for frontmatter',
|
||||
'action': 'Add closing --- after frontmatter fields',
|
||||
})
|
||||
return findings
|
||||
|
||||
frontmatter = content[4:end]
|
||||
for i, line in enumerate(frontmatter.split('\n'), start=2):
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if ':' in line:
|
||||
key = line.split(':', 1)[0].strip()
|
||||
if key not in VALID_FRONTMATTER_KEYS:
|
||||
findings.append({
|
||||
'file': filepath.name,
|
||||
'line': i,
|
||||
'severity': 'high',
|
||||
'category': 'frontmatter',
|
||||
'title': f'Invalid frontmatter key: {key}',
|
||||
'detail': f'Only {", ".join(sorted(VALID_FRONTMATTER_KEYS))} are allowed in frontmatter',
|
||||
'action': f'Remove {key} from frontmatter — use as content field in SKILL.md body instead',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def check_root_md_files(skill_path: Path) -> list[dict]:
|
||||
"""Check that no .md files exist at skill root except SKILL.md."""
|
||||
findings = []
|
||||
for md_file in skill_path.glob('*.md'):
|
||||
if md_file.name != 'SKILL.md':
|
||||
findings.append({
|
||||
'file': md_file.name,
|
||||
'line': 0,
|
||||
'severity': 'high',
|
||||
'category': 'structure',
|
||||
'title': f'Prompt file at skill root: {md_file.name}',
|
||||
'detail': 'All progressive disclosure content must be in ./references/ — only SKILL.md belongs at root',
|
||||
'action': f'Move {md_file.name} to references/{md_file.name}',
|
||||
})
|
||||
return findings
|
||||
|
||||
|
||||
def scan_file(filepath: Path, skip_fenced: bool = True) -> list[dict]:
|
||||
"""Scan a single file for path standard violations."""
|
||||
findings = []
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
rel_path = filepath.name
|
||||
|
||||
checks = [
|
||||
(DOUBLE_PREFIX_RE, 'double-prefix', 'critical',
|
||||
'Double-prefix: {project-root}/{variable} — config variables already contain {project-root} at runtime'),
|
||||
(ABSOLUTE_PATH_RE, 'absolute-path', 'high',
|
||||
'Absolute path found — not portable across machines'),
|
||||
(HOME_PATH_RE, 'absolute-path', 'high',
|
||||
'Home directory path (~/) found — environment-specific'),
|
||||
(RELATIVE_DOT_RE, 'relative-prefix', 'high',
|
||||
'Parent directory reference (../) found — fragile, breaks with reorganization'),
|
||||
(CROSS_DIR_DOT_SLASH_RE, 'cross-dir-dot-slash', 'high',
|
||||
'Cross-directory ./ reference — ./ means same folder only; use bare skill-root relative path (e.g., references/foo.md not ./references/foo.md)'),
|
||||
]
|
||||
|
||||
for pattern, category, severity, message in checks:
|
||||
for match in pattern.finditer(content):
|
||||
pos = match.start()
|
||||
if skip_fenced and is_in_fenced_block(content, pos):
|
||||
continue
|
||||
line_num = get_line_number(content, pos)
|
||||
line_content = content.split('\n')[line_num - 1].strip()
|
||||
findings.append({
|
||||
'file': rel_path,
|
||||
'line': line_num,
|
||||
'severity': severity,
|
||||
'category': category,
|
||||
'title': message,
|
||||
'detail': line_content[:120],
|
||||
'action': '',
|
||||
})
|
||||
|
||||
# Bare _bmad check — more nuanced, need to avoid false positives
|
||||
# inside {project-root}/_bmad which is correct
|
||||
for match in BARE_BMAD_RE.finditer(content):
|
||||
pos = match.start()
|
||||
if skip_fenced and is_in_fenced_block(content, pos):
|
||||
continue
|
||||
start = max(0, pos - 30)
|
||||
before = content[start:pos]
|
||||
if '{project-root}/' in before:
|
||||
continue
|
||||
line_num = get_line_number(content, pos)
|
||||
line_content = content.split('\n')[line_num - 1].strip()
|
||||
findings.append({
|
||||
'file': rel_path,
|
||||
'line': line_num,
|
||||
'severity': 'high',
|
||||
'category': 'bare-bmad',
|
||||
'title': 'Bare _bmad reference without {project-root} prefix',
|
||||
'detail': line_content[:120],
|
||||
'action': '',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def scan_skill(skill_path: Path, skip_fenced: bool = True) -> dict:
|
||||
"""Scan all .md and .json files in a skill directory."""
|
||||
all_findings = []
|
||||
|
||||
# Check for .md files at root that aren't SKILL.md
|
||||
all_findings.extend(check_root_md_files(skill_path))
|
||||
|
||||
# Check SKILL.md frontmatter
|
||||
skill_md = skill_path / 'SKILL.md'
|
||||
if skill_md.exists():
|
||||
content = skill_md.read_text(encoding='utf-8')
|
||||
all_findings.extend(check_frontmatter(content, skill_md))
|
||||
|
||||
# Find all .md and .json files
|
||||
md_files = sorted(list(skill_path.rglob('*.md')) + list(skill_path.rglob('*.json')))
|
||||
if not md_files:
|
||||
print(f"Warning: No .md or .json files found in {skill_path}", file=sys.stderr)
|
||||
|
||||
files_scanned = []
|
||||
for md_file in md_files:
|
||||
rel = md_file.relative_to(skill_path)
|
||||
files_scanned.append(str(rel))
|
||||
file_findings = scan_file(md_file, skip_fenced)
|
||||
for f in file_findings:
|
||||
f['file'] = str(rel)
|
||||
all_findings.extend(file_findings)
|
||||
|
||||
# Build summary
|
||||
by_severity = {'critical': 0, 'high': 0, 'medium': 0, 'low': 0}
|
||||
by_category = {
|
||||
'double_prefix': 0,
|
||||
'bare_bmad': 0,
|
||||
'absolute_path': 0,
|
||||
'relative_prefix': 0,
|
||||
'cross_dir_dot_slash': 0,
|
||||
'frontmatter': 0,
|
||||
'structure': 0,
|
||||
}
|
||||
|
||||
for f in all_findings:
|
||||
sev = f['severity']
|
||||
if sev in by_severity:
|
||||
by_severity[sev] += 1
|
||||
cat = f['category'].replace('-', '_')
|
||||
if cat in by_category:
|
||||
by_category[cat] += 1
|
||||
|
||||
return {
|
||||
'scanner': 'path-standards',
|
||||
'script': 'scan-path-standards.py',
|
||||
'version': '3.0.0',
|
||||
'skill_path': str(skill_path),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'files_scanned': files_scanned,
|
||||
'status': 'pass' if not all_findings else 'fail',
|
||||
'findings': all_findings,
|
||||
'assessments': {},
|
||||
'summary': {
|
||||
'total_findings': len(all_findings),
|
||||
'by_severity': by_severity,
|
||||
'by_category': by_category,
|
||||
'assessment': 'Path standards scan complete',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Scan BMad skill for path standard violations',
|
||||
)
|
||||
parser.add_argument(
|
||||
'skill_path',
|
||||
type=Path,
|
||||
help='Path to the skill directory to scan',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output', '-o',
|
||||
type=Path,
|
||||
help='Write JSON output to file instead of stdout',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--include-fenced',
|
||||
action='store_true',
|
||||
help='Also check inside fenced code blocks (by default they are skipped)',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.skill_path.is_dir():
|
||||
print(f"Error: {args.skill_path} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
result = scan_skill(args.skill_path, skip_fenced=not args.include_fenced)
|
||||
output = json.dumps(result, indent=2)
|
||||
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output)
|
||||
print(f"Results written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0 if result['status'] == 'pass' else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,745 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic scripts scanner for BMad skills.
|
||||
|
||||
Validates scripts in a skill's scripts/ folder for:
|
||||
- PEP 723 inline dependencies (Python)
|
||||
- Shebang, set -e, portability (Shell)
|
||||
- Version pinning for npx/uvx
|
||||
- Agentic design: no input(), has argparse/--help, JSON output, exit codes
|
||||
- Unit test existence
|
||||
- Over-engineering signals (line count, simple-op imports)
|
||||
- External lint: ruff (Python), shellcheck (Bash), biome (JS/TS)
|
||||
"""
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.9"
|
||||
# ///
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# External Linter Integration
|
||||
# =============================================================================
|
||||
|
||||
def _run_command(cmd: list[str], timeout: int = 30) -> tuple[int, str, str]:
|
||||
"""Run a command and return (returncode, stdout, stderr)."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, timeout=timeout,
|
||||
)
|
||||
return result.returncode, result.stdout, result.stderr
|
||||
except FileNotFoundError:
|
||||
return -1, '', f'Command not found: {cmd[0]}'
|
||||
except subprocess.TimeoutExpired:
|
||||
return -2, '', f'Command timed out after {timeout}s: {" ".join(cmd)}'
|
||||
|
||||
|
||||
def _find_uv() -> str | None:
|
||||
"""Find uv binary on PATH."""
|
||||
return shutil.which('uv')
|
||||
|
||||
|
||||
def _find_npx() -> str | None:
|
||||
"""Find npx binary on PATH."""
|
||||
return shutil.which('npx')
|
||||
|
||||
|
||||
def lint_python_ruff(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Run ruff on a Python file via uv. Returns lint findings."""
|
||||
uv = _find_uv()
|
||||
if not uv:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': 'uv not found on PATH — cannot run ruff for Python linting',
|
||||
'detail': '',
|
||||
'action': 'Install uv: https://docs.astral.sh/uv/getting-started/installation/',
|
||||
}]
|
||||
|
||||
rc, stdout, stderr = _run_command([
|
||||
uv, 'run', 'ruff', 'check', '--output-format', 'json', str(filepath),
|
||||
])
|
||||
|
||||
if rc == -1:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': f'Failed to run ruff via uv: {stderr.strip()}',
|
||||
'detail': '',
|
||||
'action': 'Ensure uv can install and run ruff: uv run ruff --version',
|
||||
}]
|
||||
|
||||
if rc == -2:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'ruff timed out on {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
# ruff outputs JSON array on stdout (even on rc=1 when issues found)
|
||||
findings = []
|
||||
try:
|
||||
issues = json.loads(stdout) if stdout.strip() else []
|
||||
except json.JSONDecodeError:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'Failed to parse ruff output for {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
for issue in issues:
|
||||
fix_msg = issue.get('fix', {}).get('message', '') if issue.get('fix') else ''
|
||||
findings.append({
|
||||
'file': rel_path,
|
||||
'line': issue.get('location', {}).get('row', 0),
|
||||
'severity': 'high',
|
||||
'category': 'lint',
|
||||
'title': f'[{issue.get("code", "?")}] {issue.get("message", "")}',
|
||||
'detail': '',
|
||||
'action': fix_msg or f'See https://docs.astral.sh/ruff/rules/{issue.get("code", "")}',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def lint_shell_shellcheck(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Run shellcheck on a shell script via uv. Returns lint findings."""
|
||||
uv = _find_uv()
|
||||
if not uv:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': 'uv not found on PATH — cannot run shellcheck for shell linting',
|
||||
'detail': '',
|
||||
'action': 'Install uv: https://docs.astral.sh/uv/getting-started/installation/',
|
||||
}]
|
||||
|
||||
rc, stdout, stderr = _run_command([
|
||||
uv, 'run', '--with', 'shellcheck-py',
|
||||
'shellcheck', '--format', 'json', str(filepath),
|
||||
])
|
||||
|
||||
if rc == -1:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': f'Failed to run shellcheck via uv: {stderr.strip()}',
|
||||
'detail': '',
|
||||
'action': 'Ensure uv can install shellcheck-py: uv run --with shellcheck-py shellcheck --version',
|
||||
}]
|
||||
|
||||
if rc == -2:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'shellcheck timed out on {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
findings = []
|
||||
# shellcheck outputs JSON on stdout (rc=1 when issues found)
|
||||
raw = stdout.strip() or stderr.strip()
|
||||
try:
|
||||
issues = json.loads(raw) if raw else []
|
||||
except json.JSONDecodeError:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'Failed to parse shellcheck output for {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
# Map shellcheck levels to our severity
|
||||
level_map = {'error': 'high', 'warning': 'high', 'info': 'high', 'style': 'medium'}
|
||||
|
||||
for issue in issues:
|
||||
sc_code = issue.get('code', '')
|
||||
findings.append({
|
||||
'file': rel_path,
|
||||
'line': issue.get('line', 0),
|
||||
'severity': level_map.get(issue.get('level', ''), 'high'),
|
||||
'category': 'lint',
|
||||
'title': f'[SC{sc_code}] {issue.get("message", "")}',
|
||||
'detail': '',
|
||||
'action': f'See https://www.shellcheck.net/wiki/SC{sc_code}',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def lint_node_biome(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Run biome on a JS/TS file via npx. Returns lint findings."""
|
||||
npx = _find_npx()
|
||||
if not npx:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': 'npx not found on PATH — cannot run biome for JS/TS linting',
|
||||
'detail': '',
|
||||
'action': 'Install Node.js 20+: https://nodejs.org/',
|
||||
}]
|
||||
|
||||
rc, stdout, stderr = _run_command([
|
||||
npx, '--yes', '@biomejs/biome', 'lint', '--reporter', 'json', str(filepath),
|
||||
], timeout=60)
|
||||
|
||||
if rc == -1:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'high', 'category': 'lint-setup',
|
||||
'title': f'Failed to run biome via npx: {stderr.strip()}',
|
||||
'detail': '',
|
||||
'action': 'Ensure npx can run biome: npx @biomejs/biome --version',
|
||||
}]
|
||||
|
||||
if rc == -2:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'biome timed out on {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
findings = []
|
||||
# biome outputs JSON on stdout
|
||||
raw = stdout.strip()
|
||||
try:
|
||||
result = json.loads(raw) if raw else {}
|
||||
except json.JSONDecodeError:
|
||||
return [{
|
||||
'file': rel_path, 'line': 0,
|
||||
'severity': 'medium', 'category': 'lint',
|
||||
'title': f'Failed to parse biome output for {rel_path}',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}]
|
||||
|
||||
for diag in result.get('diagnostics', []):
|
||||
loc = diag.get('location', {})
|
||||
start = loc.get('start', {})
|
||||
findings.append({
|
||||
'file': rel_path,
|
||||
'line': start.get('line', 0),
|
||||
'severity': 'high',
|
||||
'category': 'lint',
|
||||
'title': f'[{diag.get("category", "?")}] {diag.get("message", "")}',
|
||||
'detail': '',
|
||||
'action': diag.get('advices', [{}])[0].get('message', '') if diag.get('advices') else '',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# BMad Pattern Checks (Existing)
|
||||
# =============================================================================
|
||||
|
||||
def scan_python_script(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Check a Python script for standards compliance."""
|
||||
findings = []
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
lines = content.split('\n')
|
||||
line_count = len(lines)
|
||||
|
||||
# PEP 723 check
|
||||
if '# /// script' not in content:
|
||||
# Only flag if the script has imports (not a trivial script)
|
||||
if 'import ' in content:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'dependencies',
|
||||
'title': 'No PEP 723 inline dependency block (# /// script)',
|
||||
'detail': '',
|
||||
'action': 'Add PEP 723 block with requires-python and dependencies',
|
||||
})
|
||||
else:
|
||||
# Check requires-python is present
|
||||
if 'requires-python' not in content:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'low', 'category': 'dependencies',
|
||||
'title': 'PEP 723 block exists but missing requires-python constraint',
|
||||
'detail': '',
|
||||
'action': 'Add requires-python = ">=3.9" or appropriate version',
|
||||
})
|
||||
|
||||
# requirements.txt reference
|
||||
if 'requirements.txt' in content or 'pip install' in content:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'high', 'category': 'dependencies',
|
||||
'title': 'References requirements.txt or pip install — use PEP 723 inline deps',
|
||||
'detail': '',
|
||||
'action': 'Replace with PEP 723 inline dependency block',
|
||||
})
|
||||
|
||||
# Agentic design checks via AST
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
except SyntaxError:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'critical', 'category': 'error-handling',
|
||||
'title': 'Python syntax error — script cannot be parsed',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
})
|
||||
return findings
|
||||
|
||||
has_argparse = False
|
||||
has_json_dumps = False
|
||||
has_sys_exit = False
|
||||
imports = set()
|
||||
|
||||
for node in ast.walk(tree):
|
||||
# Track imports
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
imports.add(alias.name)
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module:
|
||||
imports.add(node.module)
|
||||
|
||||
# input() calls
|
||||
if isinstance(node, ast.Call):
|
||||
func = node.func
|
||||
if isinstance(func, ast.Name) and func.id == 'input':
|
||||
findings.append({
|
||||
'file': rel_path, 'line': node.lineno,
|
||||
'severity': 'critical', 'category': 'agentic-design',
|
||||
'title': 'input() call found — blocks in non-interactive agent execution',
|
||||
'detail': '',
|
||||
'action': 'Use argparse with required flags instead of interactive prompts',
|
||||
})
|
||||
# json.dumps
|
||||
if isinstance(func, ast.Attribute) and func.attr == 'dumps':
|
||||
has_json_dumps = True
|
||||
# sys.exit
|
||||
if isinstance(func, ast.Attribute) and func.attr == 'exit':
|
||||
has_sys_exit = True
|
||||
if isinstance(func, ast.Name) and func.id == 'exit':
|
||||
has_sys_exit = True
|
||||
|
||||
# argparse
|
||||
if isinstance(node, ast.Attribute) and node.attr == 'ArgumentParser':
|
||||
has_argparse = True
|
||||
|
||||
if not has_argparse and line_count > 20:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'agentic-design',
|
||||
'title': 'No argparse found — script lacks --help self-documentation',
|
||||
'detail': '',
|
||||
'action': 'Add argparse with description and argument help text',
|
||||
})
|
||||
|
||||
if not has_json_dumps and line_count > 20:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'agentic-design',
|
||||
'title': 'No json.dumps found — output may not be structured JSON',
|
||||
'detail': '',
|
||||
'action': 'Use json.dumps for structured output parseable by workflows',
|
||||
})
|
||||
|
||||
if not has_sys_exit and line_count > 20:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'low', 'category': 'agentic-design',
|
||||
'title': 'No sys.exit() calls — may not return meaningful exit codes',
|
||||
'detail': '',
|
||||
'action': 'Return 0=success, 1=fail, 2=error via sys.exit()',
|
||||
})
|
||||
|
||||
# Over-engineering: simple file ops in Python
|
||||
simple_op_imports = {'shutil', 'glob', 'fnmatch'}
|
||||
over_eng = imports & simple_op_imports
|
||||
if over_eng and line_count < 30:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'low', 'category': 'over-engineered',
|
||||
'title': f'Short script ({line_count} lines) imports {", ".join(over_eng)} — may be simpler as bash',
|
||||
'detail': '',
|
||||
'action': 'Consider if cp/mv/find shell commands would suffice',
|
||||
})
|
||||
|
||||
# Very short script
|
||||
if line_count < 5:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'over-engineered',
|
||||
'title': f'Script is only {line_count} lines — could be an inline command',
|
||||
'detail': '',
|
||||
'action': 'Consider inlining this command directly in the prompt',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def scan_shell_script(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Check a shell script for standards compliance."""
|
||||
findings = []
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
lines = content.split('\n')
|
||||
line_count = len(lines)
|
||||
|
||||
# Shebang
|
||||
if not lines[0].startswith('#!'):
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'high', 'category': 'portability',
|
||||
'title': 'Missing shebang line',
|
||||
'detail': '',
|
||||
'action': 'Add #!/usr/bin/env bash or #!/usr/bin/env sh',
|
||||
})
|
||||
elif '/usr/bin/env' not in lines[0]:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'portability',
|
||||
'title': f'Shebang uses hardcoded path: {lines[0].strip()}',
|
||||
'detail': '',
|
||||
'action': 'Use #!/usr/bin/env bash for cross-platform compatibility',
|
||||
})
|
||||
|
||||
# set -e
|
||||
if 'set -e' not in content and 'set -euo' not in content:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'error-handling',
|
||||
'title': 'Missing set -e — errors will be silently ignored',
|
||||
'detail': '',
|
||||
'action': 'Add set -e (or set -euo pipefail) near the top',
|
||||
})
|
||||
|
||||
# Hardcoded interpreter paths
|
||||
hardcoded_re = re.compile(r'/usr/bin/(python|ruby|node|perl)\b')
|
||||
for i, line in enumerate(lines, 1):
|
||||
if hardcoded_re.search(line):
|
||||
findings.append({
|
||||
'file': rel_path, 'line': i,
|
||||
'severity': 'medium', 'category': 'portability',
|
||||
'title': f'Hardcoded interpreter path: {line.strip()}',
|
||||
'detail': '',
|
||||
'action': 'Use /usr/bin/env or PATH-based lookup',
|
||||
})
|
||||
|
||||
# GNU-only tools
|
||||
gnu_re = re.compile(r'\b(gsed|gawk|ggrep|gfind)\b')
|
||||
for i, line in enumerate(lines, 1):
|
||||
m = gnu_re.search(line)
|
||||
if m:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': i,
|
||||
'severity': 'medium', 'category': 'portability',
|
||||
'title': f'GNU-only tool: {m.group()} — not available on all platforms',
|
||||
'detail': '',
|
||||
'action': 'Use POSIX-compatible equivalent',
|
||||
})
|
||||
|
||||
# Unquoted variables (basic check)
|
||||
unquoted_re = re.compile(r'(?<!")\$\w+(?!")')
|
||||
for i, line in enumerate(lines, 1):
|
||||
if line.strip().startswith('#'):
|
||||
continue
|
||||
for m in unquoted_re.finditer(line):
|
||||
# Skip inside double-quoted strings (rough heuristic)
|
||||
before = line[:m.start()]
|
||||
if before.count('"') % 2 == 1:
|
||||
continue
|
||||
findings.append({
|
||||
'file': rel_path, 'line': i,
|
||||
'severity': 'low', 'category': 'portability',
|
||||
'title': f'Potentially unquoted variable: {m.group()} — breaks with spaces in paths',
|
||||
'detail': '',
|
||||
'action': f'Use "{m.group()}" with double quotes',
|
||||
})
|
||||
|
||||
# npx/uvx without version pinning
|
||||
no_pin_re = re.compile(r'\b(npx|uvx)\s+([a-zA-Z][\w-]+)(?!\S*@)')
|
||||
for i, line in enumerate(lines, 1):
|
||||
if line.strip().startswith('#'):
|
||||
continue
|
||||
m = no_pin_re.search(line)
|
||||
if m:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': i,
|
||||
'severity': 'medium', 'category': 'dependencies',
|
||||
'title': f'{m.group(1)} {m.group(2)} without version pinning',
|
||||
'detail': '',
|
||||
'action': f'Pin version: {m.group(1)} {m.group(2)}@<version>',
|
||||
})
|
||||
|
||||
# Very short script
|
||||
if line_count < 5:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'over-engineered',
|
||||
'title': f'Script is only {line_count} lines — could be an inline command',
|
||||
'detail': '',
|
||||
'action': 'Consider inlining this command directly in the prompt',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
def scan_node_script(filepath: Path, rel_path: str) -> list[dict]:
|
||||
"""Check a JS/TS script for standards compliance."""
|
||||
findings = []
|
||||
content = filepath.read_text(encoding='utf-8')
|
||||
lines = content.split('\n')
|
||||
line_count = len(lines)
|
||||
|
||||
# npx/uvx without version pinning
|
||||
no_pin = re.compile(r'\b(npx|uvx)\s+([a-zA-Z][\w-]+)(?!\S*@)')
|
||||
for i, line in enumerate(lines, 1):
|
||||
m = no_pin.search(line)
|
||||
if m:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': i,
|
||||
'severity': 'medium', 'category': 'dependencies',
|
||||
'title': f'{m.group(1)} {m.group(2)} without version pinning',
|
||||
'detail': '',
|
||||
'action': f'Pin version: {m.group(1)} {m.group(2)}@<version>',
|
||||
})
|
||||
|
||||
# Very short script
|
||||
if line_count < 5:
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'over-engineered',
|
||||
'title': f'Script is only {line_count} lines — could be an inline command',
|
||||
'detail': '',
|
||||
'action': 'Consider inlining this command directly in the prompt',
|
||||
})
|
||||
|
||||
return findings
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main Scanner
|
||||
# =============================================================================
|
||||
|
||||
def scan_skill_scripts(skill_path: Path) -> dict:
|
||||
"""Scan all scripts in a skill directory."""
|
||||
scripts_dir = skill_path / 'scripts'
|
||||
all_findings = []
|
||||
lint_findings = []
|
||||
script_inventory = {'python': [], 'shell': [], 'node': [], 'other': []}
|
||||
missing_tests = []
|
||||
|
||||
if not scripts_dir.exists():
|
||||
return {
|
||||
'scanner': 'scripts',
|
||||
'script': 'scan-scripts.py',
|
||||
'version': '2.0.0',
|
||||
'skill_path': str(skill_path),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'status': 'pass',
|
||||
'findings': [{
|
||||
'file': 'scripts/',
|
||||
'severity': 'info',
|
||||
'category': 'none',
|
||||
'title': 'No scripts/ directory found — nothing to scan',
|
||||
'detail': '',
|
||||
'action': '',
|
||||
}],
|
||||
'assessments': {
|
||||
'lint_summary': {
|
||||
'tools_used': [],
|
||||
'files_linted': 0,
|
||||
'lint_issues': 0,
|
||||
},
|
||||
'script_summary': {
|
||||
'total_scripts': 0,
|
||||
'by_type': script_inventory,
|
||||
'missing_tests': [],
|
||||
},
|
||||
},
|
||||
'summary': {
|
||||
'total_findings': 0,
|
||||
'by_severity': {'critical': 0, 'high': 0, 'medium': 0, 'low': 0},
|
||||
'assessment': '',
|
||||
},
|
||||
}
|
||||
|
||||
# Find all script files (exclude tests/ and __pycache__)
|
||||
script_files = []
|
||||
for f in sorted(scripts_dir.iterdir()):
|
||||
if f.is_file() and f.suffix in ('.py', '.sh', '.bash', '.js', '.ts', '.mjs'):
|
||||
script_files.append(f)
|
||||
|
||||
tests_dir = scripts_dir / 'tests'
|
||||
lint_tools_used = set()
|
||||
|
||||
for script_file in script_files:
|
||||
rel_path = f'scripts/{script_file.name}'
|
||||
ext = script_file.suffix
|
||||
|
||||
if ext == '.py':
|
||||
script_inventory['python'].append(script_file.name)
|
||||
findings = scan_python_script(script_file, rel_path)
|
||||
lf = lint_python_ruff(script_file, rel_path)
|
||||
lint_findings.extend(lf)
|
||||
if lf and not any(f['category'] == 'lint-setup' for f in lf):
|
||||
lint_tools_used.add('ruff')
|
||||
elif ext in ('.sh', '.bash'):
|
||||
script_inventory['shell'].append(script_file.name)
|
||||
findings = scan_shell_script(script_file, rel_path)
|
||||
lf = lint_shell_shellcheck(script_file, rel_path)
|
||||
lint_findings.extend(lf)
|
||||
if lf and not any(f['category'] == 'lint-setup' for f in lf):
|
||||
lint_tools_used.add('shellcheck')
|
||||
elif ext in ('.js', '.ts', '.mjs'):
|
||||
script_inventory['node'].append(script_file.name)
|
||||
findings = scan_node_script(script_file, rel_path)
|
||||
lf = lint_node_biome(script_file, rel_path)
|
||||
lint_findings.extend(lf)
|
||||
if lf and not any(f['category'] == 'lint-setup' for f in lf):
|
||||
lint_tools_used.add('biome')
|
||||
else:
|
||||
script_inventory['other'].append(script_file.name)
|
||||
findings = []
|
||||
|
||||
# Check for unit tests
|
||||
if tests_dir.exists():
|
||||
stem = script_file.stem
|
||||
test_patterns = [
|
||||
f'test_{stem}{ext}', f'test-{stem}{ext}',
|
||||
f'{stem}_test{ext}', f'{stem}-test{ext}',
|
||||
f'test_{stem}.py', f'test-{stem}.py',
|
||||
]
|
||||
has_test = any((tests_dir / t).exists() for t in test_patterns)
|
||||
else:
|
||||
has_test = False
|
||||
|
||||
if not has_test:
|
||||
missing_tests.append(script_file.name)
|
||||
findings.append({
|
||||
'file': rel_path, 'line': 1,
|
||||
'severity': 'medium', 'category': 'tests',
|
||||
'title': f'No unit test found for {script_file.name}',
|
||||
'detail': '',
|
||||
'action': f'Create scripts/tests/test-{script_file.stem}{ext} with test cases',
|
||||
})
|
||||
|
||||
all_findings.extend(findings)
|
||||
|
||||
# Check if tests/ directory exists at all
|
||||
if script_files and not tests_dir.exists():
|
||||
all_findings.append({
|
||||
'file': 'scripts/tests/',
|
||||
'line': 0,
|
||||
'severity': 'high',
|
||||
'category': 'tests',
|
||||
'title': 'scripts/tests/ directory does not exist — no unit tests',
|
||||
'detail': '',
|
||||
'action': 'Create scripts/tests/ with test files for each script',
|
||||
})
|
||||
|
||||
# Merge lint findings into all findings
|
||||
all_findings.extend(lint_findings)
|
||||
|
||||
# Build summary
|
||||
by_severity = {'critical': 0, 'high': 0, 'medium': 0, 'low': 0}
|
||||
by_category: dict[str, int] = {}
|
||||
for f in all_findings:
|
||||
sev = f['severity']
|
||||
if sev in by_severity:
|
||||
by_severity[sev] += 1
|
||||
cat = f['category']
|
||||
by_category[cat] = by_category.get(cat, 0) + 1
|
||||
|
||||
total_scripts = sum(len(v) for v in script_inventory.values())
|
||||
status = 'pass'
|
||||
if by_severity['critical'] > 0:
|
||||
status = 'fail'
|
||||
elif by_severity['high'] > 0:
|
||||
status = 'warning'
|
||||
elif total_scripts == 0:
|
||||
status = 'pass'
|
||||
|
||||
lint_issue_count = sum(1 for f in lint_findings if f['category'] == 'lint')
|
||||
|
||||
return {
|
||||
'scanner': 'scripts',
|
||||
'script': 'scan-scripts.py',
|
||||
'version': '2.0.0',
|
||||
'skill_path': str(skill_path),
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'status': status,
|
||||
'findings': all_findings,
|
||||
'assessments': {
|
||||
'lint_summary': {
|
||||
'tools_used': sorted(lint_tools_used),
|
||||
'files_linted': total_scripts,
|
||||
'lint_issues': lint_issue_count,
|
||||
},
|
||||
'script_summary': {
|
||||
'total_scripts': total_scripts,
|
||||
'by_type': {k: len(v) for k, v in script_inventory.items()},
|
||||
'scripts': {k: v for k, v in script_inventory.items() if v},
|
||||
'missing_tests': missing_tests,
|
||||
},
|
||||
},
|
||||
'summary': {
|
||||
'total_findings': len(all_findings),
|
||||
'by_severity': by_severity,
|
||||
'by_category': by_category,
|
||||
'assessment': '',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Scan BMad skill scripts for quality, portability, agentic design, and lint issues',
|
||||
)
|
||||
parser.add_argument(
|
||||
'skill_path',
|
||||
type=Path,
|
||||
help='Path to the skill directory to scan',
|
||||
)
|
||||
parser.add_argument(
|
||||
'--output', '-o',
|
||||
type=Path,
|
||||
help='Write JSON output to file instead of stdout',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.skill_path.is_dir():
|
||||
print(f"Error: {args.skill_path} is not a directory", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
result = scan_skill_scripts(args.skill_path)
|
||||
output = json.dumps(result, indent=2)
|
||||
|
||||
if args.output:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(output)
|
||||
print(f"Results written to {args.output}", file=sys.stderr)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0 if result['status'] == 'pass' else 1
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Guard against drift between the embedded prompt-quality-canon copies.
|
||||
|
||||
The canon is embedded in three places — workflow-builder references,
|
||||
agent-builder references, and the agent-builder asset emitted into built
|
||||
agents — with no in-file sync note, because the loaded files are LLM-facing
|
||||
and a maintenance comment there is paid on every load. This test is the
|
||||
sync mechanism instead: all three copies must be byte-identical.
|
||||
Run with: python3 -m pytest test_canon_sync.py
|
||||
(or plain `python3 test_canon_sync.py` for a lightweight self-check).
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SKILLS_DIR = Path(__file__).resolve().parents[3]
|
||||
|
||||
CANON_COPIES = [
|
||||
SKILLS_DIR / "bmad-workflow-builder" / "references" / "prompt-quality-canon.md",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "references" / "prompt-quality-canon.md",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "assets" / "prompt-quality-canon.md",
|
||||
]
|
||||
|
||||
|
||||
def test_all_copies_exist():
|
||||
missing = [str(p) for p in CANON_COPIES if not p.is_file()]
|
||||
assert not missing, f"canon copy missing: {missing}"
|
||||
|
||||
|
||||
def test_all_copies_identical():
|
||||
contents = {p: p.read_bytes() for p in CANON_COPIES if p.is_file()}
|
||||
reference = CANON_COPIES[0]
|
||||
diverged = [
|
||||
str(p)
|
||||
for p, body in contents.items()
|
||||
if body != contents.get(reference)
|
||||
]
|
||||
assert not diverged, (
|
||||
"canon copies have drifted from "
|
||||
f"{reference}: {diverged} — sync all copies together"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_all_copies_exist()
|
||||
test_all_copies_identical()
|
||||
print(f"ok: {len(CANON_COPIES)} canon copies present and identical")
|
||||
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for count_tokens.py.
|
||||
|
||||
Covers the output schema, the tiktoken path and the forced-fallback path
|
||||
agreeing within tolerance, the CLI over a file and over stdin, and argument
|
||||
guards. Run with: python3 -m pytest test_count_tokens.py
|
||||
(or plain `python3 test_count_tokens.py` to run a lightweight self-check).
|
||||
"""
|
||||
import builtins
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent.parent / "count_tokens.py"
|
||||
|
||||
SAMPLE = (
|
||||
"The builder is platform-agnostic. Nothing assumes a single runtime, and no "
|
||||
"model list is ever hardcoded. Token counts replace line counts as the one "
|
||||
"length metric, with a chars-over-four fallback when tiktoken is absent.\n"
|
||||
) * 8
|
||||
|
||||
|
||||
def _load_module():
|
||||
spec = importlib.util.spec_from_file_location("count_tokens", SCRIPT)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
def test_tiktoken_path():
|
||||
mod = _load_module()
|
||||
try:
|
||||
import tiktoken # noqa: F401
|
||||
except Exception:
|
||||
# No tiktoken in this interpreter; the real path can't be exercised here.
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
assert method == "fallback"
|
||||
assert tokens == len(SAMPLE) // 4
|
||||
return
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
assert method == "tiktoken"
|
||||
assert isinstance(tokens, int)
|
||||
assert tokens > 0
|
||||
|
||||
|
||||
def test_fallback_path_when_import_blocked():
|
||||
"""Force the import of tiktoken to fail and confirm the fallback fires."""
|
||||
mod = _load_module()
|
||||
real_import = builtins.__import__
|
||||
|
||||
def blocked_import(name, *args, **kwargs):
|
||||
if name == "tiktoken" or name.startswith("tiktoken."):
|
||||
raise ImportError("blocked for test")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
builtins.__import__ = blocked_import
|
||||
try:
|
||||
tokens, method = mod.count_tokens(SAMPLE)
|
||||
finally:
|
||||
builtins.__import__ = real_import
|
||||
|
||||
assert method == "fallback"
|
||||
assert tokens == len(SAMPLE) // 4
|
||||
|
||||
|
||||
def test_paths_agree_within_tolerance():
|
||||
"""tiktoken and chars//4 should be in the same order of magnitude.
|
||||
|
||||
Skipped when tiktoken is not installed (nothing to compare against).
|
||||
"""
|
||||
mod = _load_module()
|
||||
try:
|
||||
import tiktoken # noqa: F401
|
||||
except Exception:
|
||||
return
|
||||
|
||||
real_tokens, real_method = mod.count_tokens(SAMPLE)
|
||||
assert real_method == "tiktoken"
|
||||
|
||||
fallback_tokens = len(SAMPLE) // 4
|
||||
|
||||
# The chars//4 heuristic is a rough proxy; require it within +/-50% of the
|
||||
# real count so the fallback stays a usable budget gate, not a wild guess.
|
||||
lower = real_tokens * 0.5
|
||||
upper = real_tokens * 1.5
|
||||
assert lower <= fallback_tokens <= upper, (
|
||||
f"fallback {fallback_tokens} not within 50% of tiktoken {real_tokens}"
|
||||
)
|
||||
|
||||
|
||||
def test_cli_file_output_schema(tmp_path):
|
||||
f = tmp_path / "sample.md"
|
||||
f.write_text(SAMPLE, encoding="utf-8")
|
||||
out = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), str(f)],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
assert set(data.keys()) == {"tokens", "method"}
|
||||
assert isinstance(data["tokens"], int)
|
||||
assert data["method"] in ("tiktoken", "fallback")
|
||||
assert data["tokens"] > 0
|
||||
|
||||
|
||||
def test_cli_stdin_output_schema():
|
||||
out = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--stdin"],
|
||||
input=SAMPLE, capture_output=True, text=True, check=True,
|
||||
).stdout
|
||||
data = json.loads(out)
|
||||
assert set(data.keys()) == {"tokens", "method"}
|
||||
assert isinstance(data["tokens"], int)
|
||||
assert data["method"] in ("tiktoken", "fallback")
|
||||
|
||||
|
||||
def test_cli_file_and_stdin_agree():
|
||||
"""The CLI over a file and over stdin produce the same count for same text."""
|
||||
import tempfile, os
|
||||
fd, name = tempfile.mkstemp(suffix=".md")
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(SAMPLE)
|
||||
file_out = json.loads(subprocess.run(
|
||||
[sys.executable, str(SCRIPT), name],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout)
|
||||
finally:
|
||||
os.unlink(name)
|
||||
stdin_out = json.loads(subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--stdin"],
|
||||
input=SAMPLE, capture_output=True, text=True, check=True,
|
||||
).stdout)
|
||||
assert file_out == stdin_out
|
||||
|
||||
|
||||
def test_cli_requires_an_input():
|
||||
"""No file and no --stdin is a usage error (exit 2 from argparse)."""
|
||||
res = subprocess.run(
|
||||
[sys.executable, str(SCRIPT)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
assert res.returncode != 0
|
||||
|
||||
|
||||
def _run_all():
|
||||
import tempfile
|
||||
failures = 0
|
||||
tests = [
|
||||
test_tiktoken_path,
|
||||
test_fallback_path_when_import_blocked,
|
||||
test_paths_agree_within_tolerance,
|
||||
test_cli_stdin_output_schema,
|
||||
test_cli_file_and_stdin_agree,
|
||||
test_cli_requires_an_input,
|
||||
]
|
||||
for t in tests:
|
||||
try:
|
||||
t()
|
||||
print(f"PASS {t.__name__}")
|
||||
except AssertionError as e:
|
||||
failures += 1
|
||||
print(f"FAIL {t.__name__}: {e}")
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
print(f"ERROR {t.__name__}: {e}")
|
||||
# tmp_path-based test handled separately
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
try:
|
||||
test_cli_file_output_schema(Path(d))
|
||||
print("PASS test_cli_file_output_schema")
|
||||
except Exception as e:
|
||||
failures += 1
|
||||
print(f"FAIL test_cli_file_output_schema: {e}")
|
||||
return failures
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(1 if _run_all() else 0)
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for scripts/render_report.py — the deterministic report renderer.
|
||||
|
||||
Covers: valid island injection, refusal on malformed JSON, refusal on the
|
||||
placeholder subject, the --md archival rendering, and that both shipped
|
||||
shells carry a parseable placeholder island.
|
||||
Run with: python3 -m pytest test_render_report.py
|
||||
(or plain `python3 test_render_report.py` for a lightweight self-check).
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SKILLS_DIR = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = SKILLS_DIR / "bmad-workflow-builder" / "scripts" / "render_report.py"
|
||||
SHELLS = [
|
||||
SKILLS_DIR / "bmad-workflow-builder" / "assets" / "report-shell.html",
|
||||
SKILLS_DIR / "bmad-agent-builder" / "assets" / "report-shell.html",
|
||||
]
|
||||
ISLAND_RE = re.compile(
|
||||
r'<script[^>]*\bid="report-data"[^>]*>(.*?)</script>', re.DOTALL
|
||||
)
|
||||
|
||||
VALID_DATA = {
|
||||
"schema_version": 2,
|
||||
"subject": "skills/example-skill",
|
||||
"generated": "2026-06-10",
|
||||
"verdict": "One ceremony section; otherwise sound.",
|
||||
"grade": "good",
|
||||
"summary": "Solid structure and clean wiring. The main opportunity is one over-scripted reference.",
|
||||
"standards": {
|
||||
"canon": "/abs/skills/bmad-workflow-builder/references/prompt-quality-canon.md",
|
||||
"principles": "/abs/skills/bmad-workflow-builder/references/skill-quality-principles.md",
|
||||
"scripts": "/abs/skills/bmad-workflow-builder/references/script-standards.md",
|
||||
},
|
||||
"themes": [
|
||||
{
|
||||
"title": "Scripted sequences where goals suffice",
|
||||
"root_cause": "Steps are numbered without true ordering dependencies.",
|
||||
"finding_ids": ["leanness-1"],
|
||||
"action": "Replace ordered lists with goal sentences.",
|
||||
}
|
||||
],
|
||||
"strengths": ["Frontmatter and routing map are exemplary."],
|
||||
"recommendations": [
|
||||
{"rank": 1, "action": "De-script the finalize section.", "resolves": ["leanness-1"]}
|
||||
],
|
||||
"findings": [
|
||||
{
|
||||
"id": "leanness-1",
|
||||
"lens": "leanness",
|
||||
"severity": "high",
|
||||
"title": "Numbered finalize steps are decoration",
|
||||
"location": "references/build-process.md:finalize",
|
||||
"evidence": "No step depends on a prior step's output.",
|
||||
"recommendation": "Replace with a single goal sentence.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def run_render(args):
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *[str(a) for a in args]],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_valid_island_injection():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
findings.write_text(json.dumps(VALID_DATA), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode == 0, result.stderr
|
||||
html = out.read_text(encoding="utf-8")
|
||||
|
||||
match = ISLAND_RE.search(html)
|
||||
assert match, "rendered HTML has no report-data island"
|
||||
island = json.loads(match.group(1))
|
||||
assert island["subject"] == "skills/example-skill"
|
||||
assert island["findings"][0]["id"] == "leanness-1"
|
||||
assert island["standards"]["canon"].endswith("prompt-quality-canon.md")
|
||||
assert "__PLACEHOLDER__" not in match.group(1)
|
||||
|
||||
stdout = json.loads(result.stdout)
|
||||
assert stdout["counts"] == {"critical": 0, "high": 1, "medium": 0, "low": 0}
|
||||
assert stdout["grade"] == "good"
|
||||
|
||||
|
||||
def test_refuses_bad_json():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
findings.write_text("{ this is not json", encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode != 0
|
||||
assert "not valid JSON" in result.stderr
|
||||
assert not out.exists(), "refused render must not write output"
|
||||
|
||||
|
||||
def test_refuses_placeholder_subject():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
data = dict(VALID_DATA, subject="__PLACEHOLDER__")
|
||||
findings.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out])
|
||||
assert result.returncode != 0
|
||||
assert "placeholder" in result.stderr.lower()
|
||||
assert not out.exists(), "refused render must not write output"
|
||||
|
||||
|
||||
def test_md_output():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
tmp = Path(tmp)
|
||||
findings = tmp / "findings.json"
|
||||
out = tmp / "report.html"
|
||||
md = tmp / "report.md"
|
||||
findings.write_text(json.dumps(VALID_DATA), encoding="utf-8")
|
||||
|
||||
result = run_render([findings, "--shell", SHELLS[0], "-o", out, "--md", md])
|
||||
assert result.returncode == 0, result.stderr
|
||||
text = md.read_text(encoding="utf-8")
|
||||
assert "# Analysis Report: skills/example-skill" in text
|
||||
assert "**Grade: Good**" in text
|
||||
assert "## Themes" in text
|
||||
assert "Scripted sequences where goals suffice" in text
|
||||
assert "## Strengths" in text
|
||||
assert "## Recommendations" in text
|
||||
assert "### High (1)" in text
|
||||
assert "leanness-1" in text
|
||||
|
||||
|
||||
def test_shipped_shells_carry_placeholder_island():
|
||||
for shell in SHELLS:
|
||||
match = ISLAND_RE.search(shell.read_text(encoding="utf-8"))
|
||||
assert match, f"{shell} has no report-data island"
|
||||
island = json.loads(match.group(1))
|
||||
assert island["subject"] == "__PLACEHOLDER__", (
|
||||
f"{shell} ships a non-placeholder island; a failed injection "
|
||||
"would show its contents as real findings"
|
||||
)
|
||||
assert island["findings"] == []
|
||||
|
||||
|
||||
def test_render_script_copies_identical():
|
||||
other = SKILLS_DIR / "bmad-agent-builder" / "scripts" / "render_report.py"
|
||||
assert SCRIPT.read_bytes() == other.read_bytes(), (
|
||||
"render_report.py copies have drifted between the two builder skills"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_valid_island_injection()
|
||||
test_refuses_bad_json()
|
||||
test_refuses_placeholder_subject()
|
||||
test_md_output()
|
||||
test_shipped_shells_carry_placeholder_island()
|
||||
test_render_script_copies_identical()
|
||||
print("ok: render_report tests passed")
|
||||
Reference in New Issue
Block a user