Add Bmad-Method files and some brand-design artifacts!

.agents/skills/bmad-eval-runner/scripts/aggregate_benchmark.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""Variance benchmark: summarize a metric across N runs, and compare two configs.
+
+A single skill run is noisy. Running the same case N times and summarizing the
+spread tells you whether a difference between two versions is real or just noise.
+This script computes, per numeric metric, the mean, the sample standard deviation
+(n-1, the unbiased estimator for a sample), the min, and the max across N runs.
+Given two such config summaries it reports the delta on each shared metric so a
+"did the change help" question gets a number instead of a guess.
+
+Input shapes accepted for a single config:
+  - a list of run records, each a flat dict of metric -> number
+      [{"elapsed_s": 12.1, "total_tokens": 800}, {"elapsed_s": 11.4, ...}]
+  - {"runs": [ ...records... ]}
+  - a directory of run folders, each holding timing.json files written by
+    run_evals.py (the script reads every timing.json under the directory and
+    treats each as one run record)
+
+Usage:
+  Summarize one config across its runs:
+    python3 aggregate_benchmark.py --runs CONFIG_A.json
+    python3 aggregate_benchmark.py --runs RUN_DIR/        (reads timing.json files)
+
+  Compare two configs (each summarized, then delta = B - A):
+    python3 aggregate_benchmark.py --baseline A.json --variant B.json
+
+  Self-test on a known fixture (no external input needed):
+    python3 aggregate_benchmark.py --self-test
+
+Output is one JSON object on stdout.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+from pathlib import Path
+
+
+NUMERIC = (int, float)
+
+
+# --- statistics -------------------------------------------------------------
+
+def sample_stddev(values: list[float]) -> float:
+    """Sample standard deviation using n-1 (Bessel's correction).
+
+    Returns 0.0 for fewer than two values, where the sample variance is
+    undefined and reporting zero spread is the least surprising choice.
+    """
+    n = len(values)
+    if n < 2:
+        return 0.0
+    mean = sum(values) / n
+    var = sum((x - mean) ** 2 for x in values) / (n - 1)
+    return math.sqrt(var)
+
+
+def summarize_metric(values: list[float]) -> dict:
+    return {
+        "n": len(values),
+        "mean": (sum(values) / len(values)) if values else 0.0,
+        "stddev": sample_stddev(values),
+        "min": min(values) if values else 0.0,
+        "max": max(values) if values else 0.0,
+    }
+
+
+def collect_numeric_metrics(records: list[dict]) -> dict[str, list[float]]:
+    """Group every numeric field across records by metric name."""
+    by_metric: dict[str, list[float]] = {}
+    for rec in records:
+        if not isinstance(rec, dict):
+            continue
+        for key, val in rec.items():
+            if isinstance(val, bool):
+                continue  # bools are ints in Python; not a metric
+            if isinstance(val, NUMERIC):
+                by_metric.setdefault(key, []).append(float(val))
+    return by_metric
+
+
+def summarize_config(records: list[dict]) -> dict:
+    by_metric = collect_numeric_metrics(records)
+    return {
+        "runs": len(records),
+        "metrics": {name: summarize_metric(vals)
+                    for name, vals in sorted(by_metric.items())},
+    }
+
+
+def delta_configs(baseline: dict, variant: dict) -> dict:
+    """Per shared metric, delta = variant.mean - baseline.mean, plus context."""
+    b_metrics = baseline.get("metrics", {})
+    v_metrics = variant.get("metrics", {})
+    shared = sorted(set(b_metrics) & set(v_metrics))
+    out: dict[str, dict] = {}
+    for name in shared:
+        b = b_metrics[name]
+        v = v_metrics[name]
+        diff = v["mean"] - b["mean"]
+        pct = (diff / b["mean"] * 100.0) if b["mean"] != 0 else None
+        out[name] = {
+            "baseline_mean": b["mean"],
+            "variant_mean": v["mean"],
+            "delta": diff,
+            "delta_pct": pct,
+            "baseline_stddev": b["stddev"],
+            "variant_stddev": v["stddev"],
+        }
+    return out
+
+
+# --- input loading ----------------------------------------------------------
+
+def load_records(path: Path) -> list[dict]:
+    """Load run records from a JSON file, a {'runs': [...]} file, or a dir of
+    timing.json files."""
+    if path.is_dir():
+        records: list[dict] = []
+        for f in sorted(path.rglob("timing.json")):
+            try:
+                data = json.loads(f.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError):
+                continue
+            if isinstance(data, dict):
+                records.append(data)
+        return records
+
+    data = json.loads(path.read_text(encoding="utf-8"))
+    if isinstance(data, dict) and "runs" in data:
+        data = data["runs"]
+    if not isinstance(data, list):
+        raise ValueError(f"expected a list of run records in {path}")
+    return [r for r in data if isinstance(r, dict)]
+
+
+# --- self-test --------------------------------------------------------------
+
+def run_self_test() -> int:
+    """Verify mean/stddev/min/max/delta on a known fixture."""
+    config_a = [
+        {"elapsed_s": 10.0, "total_tokens": 100},
+        {"elapsed_s": 12.0, "total_tokens": 200},
+        {"elapsed_s": 14.0, "total_tokens": 300},
+    ]
+    summary_a = summarize_config(config_a)
+    el = summary_a["metrics"]["elapsed_s"]
+    # mean of 10,12,14 = 12; n-1 stddev = sqrt(((-2)^2+0+2^2)/2)=sqrt(4)=2
+    assert el["n"] == 3, el
+    assert abs(el["mean"] - 12.0) < 1e-9, el
+    assert abs(el["stddev"] - 2.0) < 1e-9, el
+    assert el["min"] == 10.0 and el["max"] == 14.0, el
+    tok = summary_a["metrics"]["total_tokens"]
+    # mean of 100,200,300 = 200; n-1 stddev = sqrt((10000+0+10000)/2)=100
+    assert abs(tok["mean"] - 200.0) < 1e-9, tok
+    assert abs(tok["stddev"] - 100.0) < 1e-9, tok
+
+    # single value -> stddev 0
+    one = summarize_config([{"x": 5}])
+    assert one["metrics"]["x"]["stddev"] == 0.0, one
+
+    # bools are not treated as metrics
+    with_bool = summarize_config([{"ok": True, "x": 1}, {"ok": False, "x": 3}])
+    assert "ok" not in with_bool["metrics"], with_bool
+    assert abs(with_bool["metrics"]["x"]["mean"] - 2.0) < 1e-9, with_bool
+
+    # delta: variant slower by 3s on mean, faster question answered by sign
+    config_b = [
+        {"elapsed_s": 13.0, "total_tokens": 90},
+        {"elapsed_s": 15.0, "total_tokens": 110},
+        {"elapsed_s": 17.0, "total_tokens": 100},
+    ]
+    summary_b = summarize_config(config_b)
+    d = delta_configs(summary_a, summary_b)
+    # elapsed mean: A=12, B=15 -> delta +3, pct +25%
+    assert abs(d["elapsed_s"]["delta"] - 3.0) < 1e-9, d
+    assert abs(d["elapsed_s"]["delta_pct"] - 25.0) < 1e-9, d
+    # tokens mean: A=200, B=100 -> delta -100, pct -50%
+    assert abs(d["total_tokens"]["delta"] + 100.0) < 1e-9, d
+    assert abs(d["total_tokens"]["delta_pct"] + 50.0) < 1e-9, d
+
+    print(json.dumps({"self_test": "passed",
+                      "checked": ["mean", "stddev_n_minus_1", "min", "max",
+                                  "single_value_stddev", "bool_excluded",
+                                  "delta", "delta_pct"]}))
+    return 0
+
+
+# --- main -------------------------------------------------------------------
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--runs", type=Path,
+                   help="summarize one config (JSON file or dir of timing.json)")
+    p.add_argument("--baseline", type=Path,
+                   help="baseline config for a two-config comparison")
+    p.add_argument("--variant", type=Path,
+                   help="variant config for a two-config comparison")
+    p.add_argument("--self-test", action="store_true",
+                   help="run the built-in fixture self-test and exit")
+    args = p.parse_args(argv)
+
+    if args.self_test:
+        return run_self_test()
+
+    if args.baseline and args.variant:
+        b = summarize_config(load_records(args.baseline))
+        v = summarize_config(load_records(args.variant))
+        out = {
+            "baseline": b,
+            "variant": v,
+            "delta": delta_configs(b, v),
+        }
+        print(json.dumps(out, indent=2))
+        return 0
+
+    if args.runs:
+        out = summarize_config(load_records(args.runs))
+        print(json.dumps(out, indent=2))
+        return 0
+
+    p.error("provide --runs, or both --baseline and --variant, or --self-test")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval-runner/scripts/memlog.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.10"
+# ///
+"""memlog -- an append-only memory log: LLM-optimal working memory for a skill.
+
+A memlog is the dense, chronological record of everything that mattered in a piece of
+work -- every decision, direction, assumption, gap, note, and event as it happened --
+kept minimal like human memory: only what is important, never bloated. It persists
+ACROSS sessions, so a fresh session can load it once and continue. It is NOT a
+deliverable; downstream artifacts (a brief, a PRD, a report) are derived from it on
+demand.
+
+It is a FLAT log: there are no sections or grouping. Every entry is one line, recorded
+at the END in the order it happened. The chronology itself is the structure.
+
+Two invariants make it trustworthy:
+
+  1. Append-only, chronological. Entries land at the end, in the order they happen.
+     Nothing is ever inserted backward, reordered, edited, or removed. There is no
+     edit or delete subcommand by design; history is never rewritten.
+  2. Write-only / blind. Every command is an atomic, context-free write and echoes the
+     new state as one line of JSON, so the caller never re-reads the file mid-session.
+     The one time the file is read is on resume, and the caller reads it itself, not
+     via this script.
+
+Atomicity: every write goes to a temp file, is flushed and fsync'd, then atomically
+renamed over the target, so a crash never leaves a half-written entry.
+
+The file shape (.memlog.md):
+
+    ---
+    subject: Onboarding flow for a budgeting app
+    status: active
+    updated: 2026-06-06T14:22
+    ---
+
+    - (note) user picked the lean draft path
+    - (decision) lead with one pre-categorized account; defer multi-account import
+    - (direction) optimize for the anxious first-timer, not the power user
+    - (assumption) open-banking consent is available in the target market
+    - (gap) no data yet on week-1 retention baseline
+    - (event) ran baseline eval mode
+
+Each entry carries a typed tag drawn from a fixed vocabulary so the chronology stays
+machine-scannable: decision, direction, assumption, gap, note, event.
+
+Commands:
+  init         --path FILE [--field k=v ...]                create the memlog (errors if it exists)
+  append       --path FILE --type T --text STR             append one typed entry at the end
+  set-complete --path FILE                                 flip frontmatter status to complete
+
+The path is the memlog file itself (conventionally {run-folder}/.memlog.md).
+"""
+import argparse
+import json
+import os
+import sys
+from datetime import datetime
+from pathlib import Path
+
+ENTRY_TYPES = ("decision", "direction", "assumption", "gap", "note", "event")
+
+
+def now() -> str:
+    return datetime.now().strftime("%Y-%m-%dT%H:%M")
+
+
+def split(text: str) -> tuple[dict, str]:
+    """Return (frontmatter dict in source order, body str). Frontmatter is plain key: value.
+
+    The closing fence is the first line that is *exactly* `---`, so a `---` inside a
+    field value (subject is free user text) never truncates the frontmatter.
+    """
+    lines = text.splitlines()
+    if not lines or lines[0] != "---":
+        raise ValueError(".memlog.md has no frontmatter")
+    end = next((i for i in range(1, len(lines)) if lines[i] == "---"), None)
+    if end is None:
+        raise ValueError(".memlog.md frontmatter is not terminated")
+    meta: dict[str, str] = {}
+    for line in lines[1:end]:
+        if ":" in line:
+            k, v = line.split(":", 1)
+            meta[k.strip()] = v.strip()
+    return meta, "\n".join(lines[end + 1:]).lstrip("\n")
+
+
+def render(meta: dict, body: str) -> str:
+    # Neutralize newlines in values so a multi-line field can't break the fence on re-read.
+    fm = "\n".join(f"{k}: {' '.join(str(v).splitlines())}" for k, v in meta.items())
+    return "---\n" + fm + "\n---\n\n" + body.rstrip("\n") + "\n"
+
+
+def touch(meta: dict) -> None:
+    """Stamp `updated` and keep it last so the field order stays predictable."""
+    meta.pop("updated", None)
+    meta["updated"] = now()
+
+
+def write_atomic(path: Path, text: str) -> None:
+    """Temp + flush + fsync + atomic rename, so a crash never half-writes an entry."""
+    tmp = path.with_suffix(path.suffix + ".tmp")
+    with open(tmp, "w", encoding="utf-8") as f:
+        f.write(text)
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, path)
+
+
+def entry_count(body: str) -> int:
+    return sum(1 for ln in body.splitlines() if ln.startswith("- "))
+
+
+def ack(path: Path, meta: dict, body: str, entry_type: str = "") -> None:
+    """Echo new state so the caller never re-reads the file to know where it stands."""
+    out = {
+        "ok": True,
+        "memlog": str(path),
+        "status": meta.get("status", ""),
+        "n": entry_count(body),
+    }
+    if entry_type:
+        out["type"] = entry_type
+    print(json.dumps(out))
+
+
+def cmd_init(args) -> int:
+    path = Path(args.path)
+    if path.exists():
+        print(f"error: {path} already exists; use append/set-complete to update it", file=sys.stderr)
+        return 2
+    path.parent.mkdir(parents=True, exist_ok=True)
+    meta: dict[str, str] = {}
+    for pair in args.field or []:
+        if "=" not in pair:
+            print(f"error: --field expects key=value, got {pair!r}", file=sys.stderr)
+            return 2
+        k, v = pair.split("=", 1)
+        meta[k.strip()] = v.strip()
+    meta.setdefault("status", "active")
+    touch(meta)
+    write_atomic(path, render(meta, ""))
+    ack(path, meta, "")
+    return 0
+
+
+def cmd_append(args) -> int:
+    path = Path(args.path)
+    if args.type not in ENTRY_TYPES:
+        print(f"error: --type must be one of {', '.join(ENTRY_TYPES)}; got {args.type!r}", file=sys.stderr)
+        return 2
+    meta, body = split(path.read_text(encoding="utf-8"))
+    text = " ".join(args.text.split())  # collapse newlines/runs -> one-line entry
+    entry = f"- ({args.type}) {text}"
+    body = (body.rstrip("\n") + "\n" + entry) if body.strip() else entry  # always at the end
+    touch(meta)
+    write_atomic(path, render(meta, body))
+    ack(path, meta, body, args.type)
+    return 0
+
+
+def cmd_set_complete(args) -> int:
+    path = Path(args.path)
+    meta, body = split(path.read_text(encoding="utf-8"))
+    meta["status"] = "complete"
+    touch(meta)
+    write_atomic(path, render(meta, body))
+    ack(path, meta, body)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    pi = sub.add_parser("init", help="create the memlog")
+    pi.add_argument("--path", required=True, help="memlog file path (e.g. {run-folder}/.memlog.md)")
+    pi.add_argument("--field", action="append", metavar="KEY=VALUE", help="frontmatter field (repeatable)")
+    pi.set_defaults(func=cmd_init)
+
+    pa = sub.add_parser("append", help="append one typed entry at the end")
+    pa.add_argument("--path", required=True)
+    pa.add_argument("--type", required=True, choices=ENTRY_TYPES, help="entry kind")
+    pa.add_argument("--text", required=True)
+    pa.set_defaults(func=cmd_append)
+
+    pc = sub.add_parser("set-complete", help="flip frontmatter status to complete")
+    pc.add_argument("--path", required=True)
+    pc.set_defaults(func=cmd_set_complete)
+
+    args = p.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval-runner/scripts/run_evals.py

@@ -0,0 +1,615 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""Run eval cases through the configured platform adapter.
+
+A case is `input + rubric + optional state_prefix + optional files`. This
+runner does the runtime-specific part of an eval: it stages the skill under
+test and the case's fixture files into a clean working directory, builds the
+prompt the adapter understands, runs it, and records the transcript plus
+timing and token usage. Grading happens elsewhere; the grader subagent reads
+the transcript and artifacts this runner leaves behind.
+
+What this runner deliberately does NOT do:
+  - No Docker, no PTY, no keychain staging, no dual-isolation strategy.
+  - No hardcoded model. Everything runtime-specific comes from the adapter.
+
+Modes (--mode) decide which configs each case runs under:
+
+  quality  : one config, "skill" — the skill staged in the cwd.
+  baseline : two configs per case — "skill" (skill staged) and "bare"
+             (nothing staged), same input, so the bare-model floor is
+             measured under identical conditions.
+  variant  : two configs — "skill" (--skill-path) and "variant"
+             (--variant-path, the stripped or prior-version skill).
+
+Run layout: <run-dir>/<config>/<case-id>/ (plus /run-N/ when --runs > 1),
+so `aggregate_benchmark.py --baseline <run-dir>/bare --variant
+<run-dir>/skill` compares configs directly from the timing.json files.
+
+Skill staging: the skill directory is copied (symlink where possible) into
+<case-cwd>/<skill_dir>/<skill-name>/ before the adapter is invoked, where
+skill_dir comes from the adapter (default ".claude/skills"). Without this
+every config would measure the bare model.
+
+Fixtures: each path in a case's `files` list is staged into the case cwd at
+its own relative path. Sources resolve against --project-root, then the cases
+file's directory, then as absolute paths.
+
+Isolation: the subprocess env is built from scratch, never inherited. It
+holds PATH, a fresh empty HOME at <case>/.home, CLAUDE_CONFIG_DIR inside
+that HOME, the adapter's auth_env var ONLY if set non-empty in the host env
+(setting it to "" would break the runtime's own credential fallback), and any
+adapter `env_passthrough` keys present in the host env. Nothing else crosses.
+
+The adapter config file (JSON) — schema and discovery rules in
+references/platform-adapter.md, working example in
+assets/adapter-claude-code.json:
+
+  invocation      : argv template. "{prompt}" -> composed case prompt,
+                    "{cwd}" -> clean working directory.
+  auth_env        : env var name carrying auth (e.g. "ANTHROPIC_API_KEY").
+  transcript      : {"format": "stdout-jsonl"} or
+                    {"format": "file", "path": "transcript.jsonl"}.
+  skill_dir       : where the runtime discovers skills under the cwd.
+  env_passthrough : optional list of extra host env vars to forward.
+
+If no adapter config is found, the runner degrades gracefully: it stages every
+case (clean cwd, skill, fixtures, prompt with state_prefix applied) and writes
+a manifest, but records each result as "skipped: no runtime adapter
+configured" instead of crashing. A human or a configured runtime can then
+complete the run.
+
+state_prefix handling: when a case carries a state_prefix, it is PREPENDED to
+the input to place the skill mid-workflow in one shot. The composed prompt is
+recorded so the grader sees exactly what ran.
+
+Usage:
+  python3 run_evals.py \\
+    --cases CASES.json \\
+    --skill-path SKILL_DIR \\
+    --output-dir DIR \\
+    [--mode quality|baseline|variant] \\
+    [--variant-path SKILL_DIR] \\
+    [--project-root DIR] \\
+    [--adapter ADAPTER.json] \\
+    [--case-ids A1,B3] [--runs N] [--timeout SECS] [--workers N] [--quiet]
+
+CASES.json is either a list of cases or {"cases": [...]}. Each case:
+  {"id": "...", "input": "...", "rubric": [...],
+   "state_prefix": "..."?, "files": ["..."]?}
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+# --- small self-contained helpers (no Docker/keychain imports) -------------
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def new_run_id(label: str) -> str:
+    return f"{datetime.now().strftime('%Y%m%d-%H%M%S')}-{label}"
+
+
+def write_json(path: Path, data: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def read_json(path: Path) -> object:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+# --- adapter ----------------------------------------------------------------
+
+def find_adapter(explicit: Path | None, cases_file: Path) -> Path | None:
+    """Locate the adapter config. Returns None when none is configured."""
+    if explicit is not None:
+        return explicit if explicit.is_file() else None
+    env_path = os.environ.get("BMAD_EVAL_ADAPTER")
+    if env_path and Path(env_path).is_file():
+        return Path(env_path)
+    for candidate in (
+        cases_file.parent / "adapter.json",
+        cases_file.parent / ".bmad-eval-adapter.json",
+    ):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def load_adapter(path: Path) -> dict:
+    cfg = read_json(path)
+    if not isinstance(cfg, dict):
+        raise ValueError(f"adapter config must be a JSON object: {path}")
+    if "invocation" not in cfg or not isinstance(cfg["invocation"], list):
+        raise ValueError("adapter config missing 'invocation' argv list")
+    return cfg
+
+
+def build_argv(invocation: list, prompt: str, cwd: str) -> list[str]:
+    argv: list[str] = []
+    for tok in invocation:
+        tok = str(tok)
+        tok = (tok.replace("{prompt}", prompt)
+               .replace("{query}", prompt)
+               .replace("{cwd}", cwd))
+        argv.append(tok)
+    return argv
+
+
+def build_case_env(adapter: Mapping | None, home_dir: Path,
+                   host_env: Mapping[str, str]) -> dict[str, str]:
+    """Build the subprocess environment from scratch — never from os.environ.
+
+    Inheriting the host env would leak shell config, tokens, and runtime
+    state into the clean room. The env holds exactly: PATH, a fresh HOME,
+    CLAUDE_CONFIG_DIR inside it, the adapter's auth var ONLY when set
+    non-empty in the host (an empty-string auth var breaks the runtime's own
+    credential fallback), and any adapter env_passthrough keys present in
+    the host env.
+    """
+    adapter = adapter or {}
+    env = {
+        "PATH": host_env.get("PATH", ""),
+        "HOME": str(home_dir),
+        "CLAUDE_CONFIG_DIR": str(home_dir / ".claude"),
+    }
+    auth_env = adapter.get("auth_env")
+    if auth_env:
+        val = host_env.get(str(auth_env))
+        if val:
+            env[str(auth_env)] = val
+    for key in adapter.get("env_passthrough") or []:
+        val = host_env.get(str(key))
+        if val is not None:
+            env[str(key)] = val
+    return env
+
+
+# --- staging: skill under test + fixtures ------------------------------------
+
+def stage_skill(skill_path: Path, cwd: Path, skills_subdir: str) -> Path:
+    """Place the skill where the runtime discovers skills inside the cwd.
+
+    Symlink when possible (cheap, and the skill is read-only to the run);
+    copy as the fallback.
+    """
+    dest_root = cwd / skills_subdir
+    dest_root.mkdir(parents=True, exist_ok=True)
+    dest = dest_root / skill_path.name
+    if not dest.exists():
+        try:
+            os.symlink(skill_path, dest)
+        except OSError:
+            shutil.copytree(skill_path, dest, dirs_exist_ok=True)
+    return dest
+
+
+def resolve_fixtures(files: list, project_root: Path,
+                     cases_dir: Path) -> list[tuple[Path, str]]:
+    """Map each `files` entry to (source, dest-relative-path).
+
+    The entry's own relative path is preserved inside the cwd, so a bare
+    filename lands at the workspace root and a nested path keeps its
+    directory structure — matching the path the case input references.
+    """
+    out: list[tuple[Path, str]] = []
+    for entry in files or []:
+        entry = str(entry)
+        for candidate in (
+            (project_root / entry).resolve(),
+            (cases_dir / entry).resolve(),
+            Path(entry).resolve(),
+        ):
+            if candidate.is_file():
+                out.append((candidate, entry))
+                break
+        else:
+            print(f"Warning: fixture not found: {entry}", file=sys.stderr)
+    return out
+
+
+def stage_fixtures(fixtures: list[tuple[Path, str]], cwd: Path) -> None:
+    for src, dest_rel in fixtures:
+        dest = cwd / dest_rel
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+
+
+# --- case composition -------------------------------------------------------
+
+def compose_prompt(case: dict) -> str:
+    """Apply state_prefix by prepending it to the input.
+
+    The state_prefix is a bracketed prime that places the skill mid-workflow in
+    one shot. Prepending keeps the input intact and visible to the grader.
+    """
+    input_text = str(case.get("input", ""))
+    prefix = case.get("state_prefix")
+    if prefix:
+        return f"{str(prefix).rstrip()}\n\n{input_text}"
+    return input_text
+
+
+# --- transcript + token accounting -----------------------------------------
+
+def read_transcript(transcript_cfg: dict, captured_stdout: bytes,
+                    cwd: Path) -> tuple[str, str]:
+    """Return (transcript_text, source). Source names where it came from."""
+    fmt = (transcript_cfg or {}).get("format", "stdout-jsonl")
+    if fmt == "file":
+        rel = (transcript_cfg or {}).get("path", "transcript.jsonl")
+        f = cwd / rel
+        if f.is_file():
+            return f.read_text(encoding="utf-8", errors="replace"), f"file:{rel}"
+        return "", f"file:{rel} (missing)"
+    return captured_stdout.decode("utf-8", errors="replace"), "stdout"
+
+
+def account_transcript(transcript_text: str) -> dict:
+    """Pull timing/token usage from a JSONL transcript when present.
+
+    Reads usage out of the completion notification immediately, so tokens are
+    captured at run time rather than recomputed later. Recognizes the common
+    `result` event with a usage block and per-message usage blocks; unknown
+    shapes degrade to zero counts without failing.
+    """
+    input_tokens = 0
+    output_tokens = 0
+    total_steps = 0
+    tool_calls: dict[str, int] = {}
+    found_usage = False
+
+    for raw in transcript_text.splitlines():
+        raw = raw.strip()
+        if not raw:
+            continue
+        try:
+            evt = json.loads(raw)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(evt, dict):
+            continue
+        etype = evt.get("type")
+        if etype == "assistant":
+            total_steps += 1
+            msg = evt.get("message", {})
+            usage = msg.get("usage") if isinstance(msg, dict) else None
+            if isinstance(usage, dict):
+                found_usage = True
+                input_tokens += int(usage.get("input_tokens", 0) or 0)
+                output_tokens += int(usage.get("output_tokens", 0) or 0)
+            for item in (msg.get("content", []) if isinstance(msg, dict) else []):
+                if isinstance(item, dict) and item.get("type") == "tool_use":
+                    name = item.get("name", "?")
+                    tool_calls[name] = tool_calls.get(name, 0) + 1
+        elif etype == "result":
+            usage = evt.get("usage")
+            if isinstance(usage, dict):
+                found_usage = True
+                # result usage is authoritative; prefer it over the running sum
+                input_tokens = int(usage.get("input_tokens", input_tokens) or input_tokens)
+                output_tokens = int(usage.get("output_tokens", output_tokens) or output_tokens)
+
+    return {
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "total_tokens": input_tokens + output_tokens,
+        "tokens_reported": found_usage,
+        "total_steps": total_steps,
+        "tool_calls": tool_calls,
+        "total_tool_calls": sum(tool_calls.values()),
+    }
+
+
+# --- per-case execution -----------------------------------------------------
+
+def run_case(case: dict, case_dir: Path, run_dir: Path,
+             adapter: dict | None, timeout: int, config: str,
+             skill_path: Path | None,
+             fixtures: list[tuple[Path, str]]) -> dict:
+    case_id = str(case.get("id", "unnamed"))
+    cwd = case_dir / "cwd"
+    cwd.mkdir(parents=True, exist_ok=True)
+
+    stage_fixtures(fixtures, cwd)
+    if skill_path is not None:
+        skills_subdir = (adapter or {}).get("skill_dir", ".claude/skills")
+        stage_skill(skill_path, cwd, skills_subdir)
+
+    prompt = compose_prompt(case)
+    (case_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
+    write_json(case_dir / "case.json", case)
+
+    if adapter is None:
+        result = {
+            "case_id": case_id,
+            "config": config,
+            "status": "skipped",
+            "reason": "no runtime adapter configured",
+            "prompt_chars": len(prompt),
+            "cwd": str(cwd.relative_to(run_dir)),
+        }
+        write_json(case_dir / "timing.json", {
+            "case_id": case_id, "config": config, "status": "skipped",
+            "captured_at": utc_now_iso(),
+        })
+        return result
+
+    transcript_path = case_dir / "transcript.jsonl"
+    argv = build_argv(adapter["invocation"], prompt, str(cwd))
+
+    home_dir = case_dir / ".home"
+    (home_dir / ".claude").mkdir(parents=True, exist_ok=True)
+    env = build_case_env(adapter, home_dir, os.environ)
+
+    start = time.time()
+    captured = b""
+    return_code = 0
+    error_tail = ""
+    status = "ok"
+    try:
+        proc = subprocess.run(
+            argv,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=str(cwd),
+            env=env,
+            timeout=timeout,
+        )
+        captured = proc.stdout or b""
+        return_code = proc.returncode
+        error_tail = (proc.stderr or b"").decode("utf-8", errors="replace")[-2000:]
+        if return_code != 0:
+            status = "error"
+    except FileNotFoundError as e:
+        # Adapter invocation command is not on PATH: degrade, do not crash.
+        elapsed = time.time() - start
+        write_json(case_dir / "timing.json", {
+            "case_id": case_id, "config": config, "status": "adapter-missing",
+            "elapsed_s": round(elapsed, 3), "captured_at": utc_now_iso(),
+        })
+        return {
+            "case_id": case_id,
+            "config": config,
+            "status": "adapter-missing",
+            "reason": f"invocation command not found: {e}",
+            "cwd": str(cwd.relative_to(run_dir)),
+        }
+    except subprocess.TimeoutExpired as e:
+        captured = e.stdout or b""
+        return_code = -1
+        status = "timeout"
+        error_tail = f"TIMEOUT after {timeout}s"
+    elapsed = time.time() - start
+
+    transcript_text, source = read_transcript(
+        adapter.get("transcript", {}), captured, cwd
+    )
+    transcript_path.write_text(transcript_text, encoding="utf-8")
+
+    accounting = account_transcript(transcript_text)
+
+    # Capture timing/tokens immediately to timing.json (run-time snapshot).
+    timing = {
+        "case_id": case_id,
+        "config": config,
+        "status": status,
+        "elapsed_s": round(elapsed, 3),
+        "return_code": return_code,
+        "transcript_source": source,
+        "input_tokens": accounting["input_tokens"],
+        "output_tokens": accounting["output_tokens"],
+        "total_tokens": accounting["total_tokens"],
+        "tokens_reported": accounting["tokens_reported"],
+        "total_steps": accounting["total_steps"],
+        "total_tool_calls": accounting["total_tool_calls"],
+        "captured_at": utc_now_iso(),
+    }
+    write_json(case_dir / "timing.json", timing)
+
+    return {
+        "case_id": case_id,
+        "config": config,
+        "status": status,
+        "elapsed_s": round(elapsed, 3),
+        "return_code": return_code,
+        "transcript": str(transcript_path.relative_to(run_dir)),
+        "cwd": str(cwd.relative_to(run_dir)),
+        "tokens": accounting["total_tokens"],
+        "tool_calls": accounting["tool_calls"],
+        "error_tail": error_tail,
+    }
+
+
+# --- main -------------------------------------------------------------------
+
+def load_cases(cases_file: Path) -> list[dict]:
+    data = read_json(cases_file)
+    if isinstance(data, dict) and "cases" in data:
+        cases = data["cases"]
+    elif isinstance(data, list):
+        cases = data
+    else:
+        raise ValueError("cases file must be a list or {'cases': [...]}")
+    if not isinstance(cases, list):
+        raise ValueError("'cases' must be a list")
+    return cases
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--cases", required=True, type=Path)
+    p.add_argument("--skill-path", required=True, type=Path,
+                   help="directory of the skill under test (contains SKILL.md)")
+    p.add_argument("--output-dir", required=True, type=Path)
+    p.add_argument("--mode", choices=("quality", "baseline", "variant"),
+                   default="quality")
+    p.add_argument("--variant-path", type=Path, default=None,
+                   help="variant mode: the stripped or prior-version skill")
+    p.add_argument("--project-root", type=Path, default=None,
+                   help="base for resolving fixture paths; defaults to the "
+                        "cases file's directory")
+    p.add_argument("--adapter", type=Path, default=None,
+                   help="adapter config JSON; defaults to BMAD_EVAL_ADAPTER env "
+                        "or adapter.json beside the cases file")
+    p.add_argument("--case-ids", default=None,
+                   help="comma-separated subset of case ids to run")
+    p.add_argument("--runs", type=int, default=1,
+                   help="repeats per case per config for the variance benchmark")
+    p.add_argument("--timeout", type=int, default=600)
+    p.add_argument("--workers", type=int, default=4)
+    p.add_argument("--label", default="evals", help="label for the run id")
+    p.add_argument("--quiet", action="store_true")
+    args = p.parse_args(argv)
+
+    cases_file = args.cases.resolve()
+    if not cases_file.is_file():
+        print(f"cases file not found: {cases_file}", file=sys.stderr)
+        return 2
+
+    skill_path = args.skill_path.resolve()
+    if not (skill_path / "SKILL.md").is_file():
+        print(f"skill path has no SKILL.md: {skill_path}", file=sys.stderr)
+        return 2
+
+    if args.mode == "variant":
+        if args.variant_path is None:
+            print("--mode variant requires --variant-path", file=sys.stderr)
+            return 2
+        variant_path = args.variant_path.resolve()
+        if not (variant_path / "SKILL.md").is_file():
+            print(f"variant path has no SKILL.md: {variant_path}",
+                  file=sys.stderr)
+            return 2
+    else:
+        variant_path = None
+
+    project_root = (args.project_root.resolve() if args.project_root
+                    else cases_file.parent)
+
+    # Each config is (name, skill-to-stage-or-None). Baseline runs every case
+    # twice — skill staged and bare — so the floor is measured under
+    # identical conditions.
+    if args.mode == "baseline":
+        configs: list[tuple[str, Path | None]] = [
+            ("skill", skill_path), ("bare", None)]
+    elif args.mode == "variant":
+        configs = [("skill", skill_path), ("variant", variant_path)]
+    else:
+        configs = [("skill", skill_path)]
+
+    cases = load_cases(cases_file)
+    if args.case_ids:
+        wanted = {x.strip() for x in args.case_ids.split(",") if x.strip()}
+        cases = [c for c in cases if str(c.get("id")) in wanted]
+
+    adapter_path = find_adapter(args.adapter, cases_file)
+    adapter: dict | None = None
+    adapter_note = "none"
+    if adapter_path is not None:
+        try:
+            adapter = load_adapter(adapter_path)
+            adapter_note = str(adapter_path)
+        except Exception as e:
+            print(f"adapter config invalid ({e}); degrading to skip-only",
+                  file=sys.stderr)
+            adapter = None
+            adapter_note = f"invalid: {e}"
+
+    run_id = new_run_id(args.label)
+    run_dir = (args.output_dir / run_id).resolve()
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    write_json(run_dir / "run.json", {
+        "run_id": run_id,
+        "cases_file": str(cases_file),
+        "skill_path": str(skill_path),
+        "variant_path": str(variant_path) if variant_path else None,
+        "mode": args.mode,
+        "configs": [name for name, _ in configs],
+        "runs_per_case": args.runs,
+        "adapter": adapter_note,
+        "started_at": utc_now_iso(),
+        "case_count": len(cases),
+    })
+
+    if adapter is None and not args.quiet:
+        print("[run_evals] no runtime adapter configured; staging cases only "
+              "(no crash). Configure an adapter to execute.", file=sys.stderr)
+
+    results: list[dict] = []
+    if not args.quiet:
+        print(f"[run_evals] {len(cases)} cases x {len(configs)} configs x "
+              f"{args.runs} runs, mode={args.mode}, run_dir={run_dir}",
+              file=sys.stderr)
+
+    jobs: list[tuple[str, dict, Path, Path | None]] = []
+    for config_name, config_skill in configs:
+        for c in cases:
+            base = run_dir / config_name / str(c.get("id", "unnamed"))
+            for i in range(max(1, args.runs)):
+                case_dir = base / f"run-{i + 1}" if args.runs > 1 else base
+                jobs.append((config_name, c, case_dir, config_skill))
+
+    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
+        fut_to_case = {
+            pool.submit(run_case, c, case_dir, run_dir, adapter,
+                        int(c.get("timeout", args.timeout)), config_name,
+                        config_skill,
+                        resolve_fixtures(c.get("files", []), project_root,
+                                         cases_file.parent)): c
+            for config_name, c, case_dir, config_skill in jobs
+        }
+        for fut in as_completed(fut_to_case):
+            c = fut_to_case[fut]
+            try:
+                res = fut.result()
+            except Exception as e:
+                res = {"case_id": str(c.get("id")), "status": "exception",
+                       "reason": str(e)}
+            results.append(res)
+            if not args.quiet:
+                print(f"  [{res.get('status')}] {res.get('config', '?')}/"
+                      f"{res.get('case_id')} ({res.get('elapsed_s', 0)}s)",
+                      file=sys.stderr)
+
+    summary = {
+        "run_id": run_id,
+        "completed_at": utc_now_iso(),
+        "mode": args.mode,
+        "total": len(jobs),
+        "executed": sum(1 for r in results if r.get("status") == "ok"),
+        "skipped": sum(1 for r in results if r.get("status") == "skipped"),
+        "failures": sum(1 for r in results
+                        if r.get("status") in ("error", "timeout", "exception",
+                                               "adapter-missing")),
+        "run_dir": str(run_dir),
+        "results": results,
+    }
+    write_json(run_dir / "execution-summary.json", summary)
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval-runner/scripts/run_triggers.py

@@ -0,0 +1,462 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""Trigger evals: does a skill's description fire on each near-miss query?
+
+A trigger query is a should/should-not user message that shares keywords with
+the skill so the description has to discriminate. For each query the runner
+stages a synthetic skill where the runtime looks for skills, sends the query
+through the adapter, and detects whether the skill loaded. Each query runs
+several times (runs-per-query) so the trigger rate is stable, not a coin flip.
+
+Detection lives behind the adapter. "Did the skill load" is a runtime-specific
+signal, so the adapter declares how skills are staged and how a load shows up in
+the transcript. The adapter config (see references/platform-adapter.md) adds two
+trigger-specific keys to the core ones:
+
+  invocation : argv template; "{prompt}" (or "{query}") is replaced with the
+               query text, "{cwd}" with the staging dir.
+  auth_env   : auth env-var name, forwarded only when set non-empty on the
+               host. No model id.
+  skill_dir  : path under the staging cwd where a skill is discovered, e.g.
+               ".claude/skills". The runner writes the synthetic skill there.
+  load_signal: which tool_use events count as a load:
+                 {"skill_tool": "Skill", "read_tool": "Read"}  (defaults)
+               A load is a tool_use of skill_tool whose input names the
+               synthetic skill, or a read_tool whose file_path falls inside
+               the synthetic skill's directory. Whole-transcript substring
+               matching is NOT supported: the runtime's init event lists
+               every discovered skill, so a substring match reports 100%
+               trigger rate regardless of the description.
+
+Each query runs in a built-from-scratch environment (PATH, fresh empty HOME,
+CLAUDE_CONFIG_DIR inside it, auth var only when set, adapter env_passthrough
+keys) so the host's installed skills, memory, and config cannot bias firing.
+
+If no adapter is configured the runner degrades gracefully: it stages each query
+and records "skipped: no runtime adapter configured" rather than crashing.
+
+Usage:
+  python3 run_triggers.py \\
+    --skill-path SKILL_DIR \\
+    --queries QUERIES.json \\
+    --output-dir DIR \\
+    [--adapter ADAPTER.json] \\
+    [--runs-per-query N] [--threshold 0.5] [--timeout SECS] \\
+    [--workers N] [--quiet]
+
+QUERIES.json is a list of {"query": "...", "should_trigger": true|false}.
+SKILL_DIR contains the SKILL.md whose name + description are under test; the
+description is what the synthetic skill advertises.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+# --- self-contained helpers -------------------------------------------------
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def new_run_id(label: str) -> str:
+    return f"{datetime.now().strftime('%Y%m%d-%H%M%S')}-{label}"
+
+
+def write_json(path: Path, data: object) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def read_json(path: Path) -> object:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def parse_skill_md(skill_path: Path) -> tuple[str, str]:
+    """Return (name, description) from SKILL.md frontmatter."""
+    text = (skill_path / "SKILL.md").read_text(encoding="utf-8")
+    m = re.match(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL)
+    if not m:
+        raise ValueError(f"SKILL.md at {skill_path} is missing frontmatter")
+    frontmatter = m.group(1)
+    name = None
+    desc_lines: list[str] = []
+    in_desc = False
+    for line in frontmatter.splitlines():
+        if line.startswith("name:"):
+            name = line.split(":", 1)[1].strip()
+            in_desc = False
+        elif line.startswith("description:"):
+            value = line.split(":", 1)[1].strip()
+            if value in ("|", ">"):
+                in_desc = True
+            else:
+                desc_lines = [value]
+                in_desc = False
+        elif in_desc and line.startswith(("  ", "\t")):
+            desc_lines.append(line.strip())
+        elif in_desc:
+            in_desc = False
+    if not name:
+        raise ValueError(f"SKILL.md at {skill_path} has no name")
+    return name, " ".join(desc_lines).strip()
+
+
+# --- adapter ----------------------------------------------------------------
+
+def find_adapter(explicit: Path | None, queries_file: Path) -> Path | None:
+    if explicit is not None:
+        return explicit if explicit.is_file() else None
+    env_path = os.environ.get("BMAD_EVAL_ADAPTER")
+    if env_path and Path(env_path).is_file():
+        return Path(env_path)
+    for candidate in (
+        queries_file.parent / "adapter.json",
+        queries_file.parent / ".bmad-eval-adapter.json",
+    ):
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def load_adapter(path: Path) -> dict:
+    cfg = read_json(path)
+    if not isinstance(cfg, dict) or "invocation" not in cfg:
+        raise ValueError(f"adapter config missing 'invocation': {path}")
+    return cfg
+
+
+def build_argv(invocation: list, query: str, cwd: str) -> list[str]:
+    out: list[str] = []
+    for tok in invocation:
+        tok = (str(tok).replace("{prompt}", query)
+               .replace("{query}", query)
+               .replace("{cwd}", cwd))
+        out.append(tok)
+    return out
+
+
+def build_case_env(adapter: dict | None, home_dir: Path,
+                   host_env: dict) -> dict[str, str]:
+    """Build the subprocess environment from scratch — never from os.environ.
+
+    Inheriting the host env would leak shell config, tokens, and runtime
+    state into the clean room. The env holds exactly: PATH, a fresh HOME,
+    CLAUDE_CONFIG_DIR inside it, the adapter's auth var ONLY when set
+    non-empty in the host (an empty-string auth var breaks the runtime's own
+    credential fallback), and any adapter env_passthrough keys present in
+    the host env.
+    """
+    adapter = adapter or {}
+    env = {
+        "PATH": host_env.get("PATH", ""),
+        "HOME": str(home_dir),
+        "CLAUDE_CONFIG_DIR": str(home_dir / ".claude"),
+    }
+    auth_env = adapter.get("auth_env")
+    if auth_env:
+        val = host_env.get(str(auth_env))
+        if val:
+            env[str(auth_env)] = val
+    for key in adapter.get("env_passthrough") or []:
+        val = host_env.get(str(key))
+        if val is not None:
+            env[str(key)] = val
+    return env
+
+
+# --- synthetic skill staging ------------------------------------------------
+
+def write_synthetic_skill(skills_dir: Path, skill_name: str,
+                          description: str, unique: str) -> str:
+    """Write a synthetic skill the runtime can discover. Returns its unique name.
+
+    A unique suffix lets the detector tell this synthetic skill apart from any
+    real skill of the same display name.
+    """
+    clean_name = f"{skill_name}-trig-{unique}"
+    root = skills_dir / clean_name
+    root.mkdir(parents=True, exist_ok=True)
+    indented = "\n  ".join(description.split("\n"))
+    (root / "SKILL.md").write_text(
+        f"---\n"
+        f"name: {clean_name}\n"
+        f"description: |\n"
+        f"  {indented}\n"
+        f"---\n\n"
+        f"# {skill_name}\n\n"
+        f"This skill handles: {description}\n",
+        encoding="utf-8",
+    )
+    return clean_name
+
+
+# --- load detection (behind the adapter) ------------------------------------
+
+def validate_load_signal(load_signal: dict | None) -> None:
+    """Reject substring-style load signals before any query runs."""
+    if (load_signal or {}).get("type") == "string":
+        raise ValueError(
+            "load_signal type 'string' is not supported: the runtime's init "
+            "event lists every discovered skill, so a whole-transcript "
+            "substring match reports 100% trigger rate regardless of the "
+            "description. Use tool-call detection "
+            '({"skill_tool": ..., "read_tool": ...}).'
+        )
+
+
+def detect_load(transcript_text: str, load_signal: dict, clean_name: str) -> bool:
+    """Did the synthetic skill load? Only tool_use events count.
+
+    The init event of a stream-json transcript lists every discovered skill
+    by name, so the name appearing somewhere in the transcript proves
+    nothing. A load is a skill-invocation tool call naming the synthetic
+    skill, or a read of a file inside the synthetic skill's directory (its
+    SKILL.md) — the two ways a runtime actually pulls a skill into context.
+    """
+    validate_load_signal(load_signal)
+    sig = load_signal or {}
+    skill_tool = sig.get("skill_tool", "Skill")
+    read_tool = sig.get("read_tool", "Read")
+
+    for raw in transcript_text.splitlines():
+        raw = raw.strip()
+        if not raw:
+            continue
+        try:
+            evt = json.loads(raw)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(evt, dict) or evt.get("type") != "assistant":
+            continue
+        msg = evt.get("message", {})
+        content = msg.get("content", []) if isinstance(msg, dict) else []
+        for item in content:
+            if not isinstance(item, dict) or item.get("type") != "tool_use":
+                continue
+            name = item.get("name")
+            inp = item.get("input", {})
+            if not isinstance(inp, dict):
+                inp = {}
+            if name == skill_tool and clean_name in json.dumps(inp):
+                return True
+            if name == read_tool and clean_name in str(inp.get("file_path", "")):
+                return True
+    return False
+
+
+# --- per-query execution ----------------------------------------------------
+
+def run_query_once(query: str, skill_name: str, description: str,
+                   adapter: dict, stage_dir: Path, timeout: int) -> bool:
+    skill_subdir = adapter.get("skill_dir", ".claude/skills")
+    skills_dir = stage_dir / skill_subdir
+    skills_dir.mkdir(parents=True, exist_ok=True)
+    unique = uuid.uuid4().hex[:8]
+    clean_name = write_synthetic_skill(skills_dir, skill_name, description, unique)
+
+    home_dir = stage_dir / ".home"
+    (home_dir / ".claude").mkdir(parents=True, exist_ok=True)
+    env = build_case_env(adapter, home_dir, dict(os.environ))
+
+    argv = build_argv(adapter["invocation"], query, str(stage_dir))
+    try:
+        proc = subprocess.run(
+            argv,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            cwd=str(stage_dir),
+            env=env,
+            timeout=timeout,
+        )
+        captured = proc.stdout or b""
+    except subprocess.TimeoutExpired as e:
+        captured = e.stdout or b""
+    except FileNotFoundError:
+        # invocation command absent; treat as undetected and let caller note it
+        raise
+
+    transcript_cfg = adapter.get("transcript", {"format": "stdout-jsonl"})
+    if transcript_cfg.get("format") == "file":
+        f = stage_dir / transcript_cfg.get("path", "transcript.jsonl")
+        text = f.read_text(encoding="utf-8", errors="replace") if f.is_file() else ""
+    else:
+        text = captured.decode("utf-8", errors="replace")
+
+    return detect_load(text, adapter.get("load_signal", {}), clean_name)
+
+
+# --- main -------------------------------------------------------------------
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--skill-path", required=True, type=Path)
+    p.add_argument("--queries", required=True, type=Path)
+    p.add_argument("--output-dir", required=True, type=Path)
+    p.add_argument("--adapter", type=Path, default=None)
+    p.add_argument("--runs-per-query", type=int, default=3)
+    p.add_argument("--threshold", type=float, default=0.5)
+    p.add_argument("--timeout", type=int, default=60)
+    p.add_argument("--workers", type=int, default=4)
+    p.add_argument("--quiet", action="store_true")
+    args = p.parse_args(argv)
+
+    skill_path = args.skill_path.resolve()
+    queries_file = args.queries.resolve()
+    if not queries_file.is_file():
+        print(f"queries file not found: {queries_file}", file=sys.stderr)
+        return 2
+
+    skill_name, description = parse_skill_md(skill_path)
+    queries = read_json(queries_file)
+    if not isinstance(queries, list):
+        print("queries file must be a JSON list", file=sys.stderr)
+        return 2
+
+    adapter_path = find_adapter(args.adapter, queries_file)
+    adapter: dict | None = None
+    adapter_note = "none"
+    if adapter_path is not None:
+        try:
+            adapter = load_adapter(adapter_path)
+            validate_load_signal(adapter.get("load_signal"))
+            adapter_note = str(adapter_path)
+        except Exception as e:
+            print(f"adapter config invalid ({e}); degrading to skip-only",
+                  file=sys.stderr)
+            adapter = None
+            adapter_note = f"invalid: {e}"
+
+    run_id = new_run_id(f"{skill_name}-triggers")
+    run_dir = (args.output_dir / run_id).resolve()
+    (run_dir / "queries").mkdir(parents=True, exist_ok=True)
+
+    write_json(run_dir / "run.json", {
+        "run_id": run_id,
+        "skill_name": skill_name,
+        "description": description,
+        "adapter": adapter_note,
+        "started_at": utc_now_iso(),
+        "query_count": len(queries),
+        "runs_per_query": args.runs_per_query,
+        "threshold": args.threshold,
+    })
+
+    if adapter is None:
+        if not args.quiet:
+            print("[run_triggers] no runtime adapter configured; staging only "
+                  "(no crash).", file=sys.stderr)
+        output = {
+            "run_id": run_id,
+            "completed_at": utc_now_iso(),
+            "skill_name": skill_name,
+            "description": description,
+            "status": "skipped",
+            "reason": "no runtime adapter configured",
+            "results": [],
+            "summary": {"total": len(queries), "passed": 0, "failed": 0,
+                        "skipped": len(queries)},
+        }
+        write_json(run_dir / "triggers-result.json", output)
+        print(json.dumps(output, indent=2))
+        return 0
+
+    adapter_missing = {"flag": False}
+
+    def run_one(idx: int, q: dict, run_idx: int) -> tuple[int, bool]:
+        stage = run_dir / "queries" / f"q{idx:03d}-r{run_idx}"
+        stage.mkdir(parents=True, exist_ok=True)
+        try:
+            triggered = run_query_once(
+                q["query"], skill_name, description, adapter, stage, args.timeout)
+        except FileNotFoundError:
+            adapter_missing["flag"] = True
+            triggered = False
+        finally:
+            shutil.rmtree(stage / adapter.get("skill_dir", ".claude/skills").split("/")[0],
+                          ignore_errors=True)
+        return idx, triggered
+
+    per_query: dict[int, list[bool]] = {}
+    if not args.quiet:
+        print(f"[run_triggers] {len(queries)} queries x {args.runs_per_query} "
+              f"runs", file=sys.stderr)
+
+    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool:
+        futures = []
+        for idx, q in enumerate(queries):
+            for run_idx in range(args.runs_per_query):
+                futures.append(pool.submit(run_one, idx, q, run_idx))
+        for fut in as_completed(futures):
+            try:
+                idx, triggered = fut.result()
+            except Exception as e:
+                print(f"Warning: query run failed: {e}", file=sys.stderr)
+                continue
+            per_query.setdefault(idx, []).append(triggered)
+
+    if adapter_missing["flag"]:
+        output = {
+            "run_id": run_id,
+            "completed_at": utc_now_iso(),
+            "skill_name": skill_name,
+            "status": "adapter-missing",
+            "reason": "adapter invocation command not found on PATH",
+            "results": [],
+            "summary": {"total": len(queries), "passed": 0, "failed": 0},
+        }
+        write_json(run_dir / "triggers-result.json", output)
+        print(json.dumps(output, indent=2))
+        return 0
+
+    results = []
+    for idx, q in enumerate(queries):
+        runs = per_query.get(idx, [])
+        rate = (sum(runs) / len(runs)) if runs else 0.0
+        should = bool(q.get("should_trigger", True))
+        passed = (rate >= args.threshold) if should else (rate < args.threshold)
+        results.append({
+            "query": q["query"],
+            "should_trigger": should,
+            "trigger_rate": round(rate, 3),
+            "triggers": int(sum(runs)),
+            "runs": len(runs),
+            "pass": passed,
+        })
+
+    output = {
+        "run_id": run_id,
+        "completed_at": utc_now_iso(),
+        "skill_name": skill_name,
+        "description": description,
+        "adapter": adapter_note,
+        "results": results,
+        "summary": {
+            "total": len(results),
+            "passed": sum(1 for r in results if r["pass"]),
+            "failed": sum(1 for r in results if not r["pass"]),
+        },
+    }
+    write_json(run_dir / "triggers-result.json", output)
+    print(json.dumps(output, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.agents/skills/bmad-eval-runner/scripts/tests/test_env_isolation.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Guard the clean-room env contract in run_evals.py and run_triggers.py.
+
+The eval result is only honest if nothing from the host shell leaks into the
+subprocess. Both scripts carry their own build_case_env (they are
+deliberately self-contained); this test pins the contract on both copies:
+exactly PATH + fresh HOME + CLAUDE_CONFIG_DIR + auth-var-only-when-set +
+declared passthrough keys, nothing else.
+Run with: python3 -m pytest test_env_isolation.py
+(or plain `python3 test_env_isolation.py` for a lightweight self-check).
+"""
+import sys
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+import run_evals  # noqa: E402
+import run_triggers  # noqa: E402
+
+BUILDERS = [run_evals.build_case_env, run_triggers.build_case_env]
+
+HOST_ENV = {
+    "PATH": "/usr/bin:/bin",
+    "HOME": "/Users/host",
+    "ANTHROPIC_API_KEY": "sk-test-123",
+    "AWS_SECRET_ACCESS_KEY": "host-secret-must-not-leak",
+    "CLAUDE_CONFIG_DIR": "/Users/host/.claude",
+    "EXTRA_VAR": "extra",
+}
+
+HOME = Path("/tmp/eval-case/.home")
+
+
+def test_minimal_env_keys():
+    adapter = {"auth_env": "ANTHROPIC_API_KEY"}
+    for build in BUILDERS:
+        env = build(adapter, HOME, HOST_ENV)
+        assert set(env) == {"PATH", "HOME", "CLAUDE_CONFIG_DIR",
+                            "ANTHROPIC_API_KEY"}, (build.__module__, env)
+        assert env["PATH"] == HOST_ENV["PATH"]
+        assert env["HOME"] == str(HOME), "HOME must be the fresh case home"
+        assert env["CLAUDE_CONFIG_DIR"] == str(HOME / ".claude")
+        assert env["ANTHROPIC_API_KEY"] == "sk-test-123"
+        assert "AWS_SECRET_ACCESS_KEY" not in env, "host secrets leaked"
+
+
+def test_auth_var_absent_when_unset():
+    # Setting auth to "" breaks the runtime's OAuth fallback — the key must
+    # be absent, never empty.
+    adapter = {"auth_env": "ANTHROPIC_API_KEY"}
+    for host in ({}, {"ANTHROPIC_API_KEY": ""}):
+        for build in BUILDERS:
+            env = build(adapter, HOME, {"PATH": "/bin", **host})
+            assert "ANTHROPIC_API_KEY" not in env, (build.__module__, env)
+
+
+def test_no_adapter_still_minimal():
+    for build in BUILDERS:
+        env = build(None, HOME, HOST_ENV)
+        assert set(env) == {"PATH", "HOME", "CLAUDE_CONFIG_DIR"}, env
+
+
+def test_env_passthrough_only_declared_and_present():
+    adapter = {"auth_env": "ANTHROPIC_API_KEY",
+               "env_passthrough": ["EXTRA_VAR", "NOT_SET_ON_HOST"]}
+    for build in BUILDERS:
+        env = build(adapter, HOME, HOST_ENV)
+        assert env.get("EXTRA_VAR") == "extra"
+        assert "NOT_SET_ON_HOST" not in env
+        assert "AWS_SECRET_ACCESS_KEY" not in env
+
+
+if __name__ == "__main__":
+    test_minimal_env_keys()
+    test_auth_var_absent_when_unset()
+    test_no_adapter_still_minimal()
+    test_env_passthrough_only_declared_and_present()
+    print("ok: build_case_env contract holds in run_evals and run_triggers")

.agents/skills/bmad-eval-runner/scripts/tests/test_trigger_detection.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Guard trigger detection in run_triggers.py.
+
+The stream-json init event lists every discovered skill by name, so any
+detection that substring-matches the whole transcript reports a 100% trigger
+rate. These tests pin the rule: only tool_use events (a Skill call naming the
+synthetic skill, or a Read inside its directory) count as a load, and
+substring-style load signals are rejected outright.
+Run with: python3 -m pytest test_trigger_detection.py
+(or plain `python3 test_trigger_detection.py` for a lightweight self-check).
+"""
+import json
+import sys
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from run_triggers import detect_load, validate_load_signal  # noqa: E402
+
+NAME = "my-skill-trig-abc12345"
+
+
+def line(obj) -> str:
+    return json.dumps(obj)
+
+
+def init_event() -> str:
+    # Claude Code's init event advertises every discovered skill.
+    return line({"type": "system", "subtype": "init",
+                 "tools": ["Skill", "Read", "Bash"],
+                 "slash_commands": [], "skills": [NAME, "other-skill"]})
+
+
+def assistant(content) -> str:
+    return line({"type": "assistant", "message": {"content": content}})
+
+
+def test_init_event_alone_is_not_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "text", "text": "I can't help with that."}]),
+        line({"type": "result", "usage": {}}),
+    ])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_text_mention_is_not_a_load():
+    transcript = assistant(
+        [{"type": "text", "text": f"There is a skill called {NAME} available."}])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_skill_tool_call_is_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Skill",
+                    "input": {"skill": NAME}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is True
+
+
+def test_read_of_synthetic_skill_md_is_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Read",
+                    "input": {"file_path":
+                              f"/tmp/stage/.claude/skills/{NAME}/SKILL.md"}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is True
+
+
+def test_unrelated_tool_calls_are_not_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Read",
+                    "input": {"file_path": "/tmp/stage/notes.md"}}]),
+        assistant([{"type": "tool_use", "name": "Skill",
+                    "input": {"skill": "other-skill"}}]),
+        assistant([{"type": "tool_use", "name": "Bash",
+                    "input": {"command": f"echo {NAME}"}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_custom_tool_names_from_load_signal():
+    sig = {"skill_tool": "InvokeSkill", "read_tool": "OpenFile"}
+    hit = assistant([{"type": "tool_use", "name": "InvokeSkill",
+                      "input": {"name": NAME}}])
+    miss = assistant([{"type": "tool_use", "name": "Skill",
+                       "input": {"skill": NAME}}])
+    assert detect_load(hit, sig, NAME) is True
+    assert detect_load(miss, sig, NAME) is False, \
+        "default tool name must not fire when the adapter renames it"
+
+
+def test_garbage_lines_do_not_crash():
+    transcript = "not json\n\n{\"type\": 42}\n[1,2,3]\n"
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_string_load_signal_rejected():
+    for fn, args in ((validate_load_signal, ({"type": "string"},)),
+                     (detect_load, ("", {"type": "string"}, NAME))):
+        try:
+            fn(*args)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError(
+                f"{fn.__name__} accepted a substring load_signal")
+
+
+if __name__ == "__main__":
+    test_init_event_alone_is_not_a_load()
+    test_text_mention_is_not_a_load()
+    test_skill_tool_call_is_a_load()
+    test_read_of_synthetic_skill_md_is_a_load()
+    test_unrelated_tool_calls_are_not_a_load()
+    test_custom_tool_names_from_load_signal()
+    test_garbage_lines_do_not_crash()
+    test_string_load_signal_rejected()
+    print("ok: trigger detection counts tool calls only; substring rejected")