Add Bmad-Method files and some brand-design artifacts!

2026-07-06 18:00:53 +01:00
parent 87bff9c251
commit 2b4a6a413c
1806 changed files with 417144 additions and 0 deletions
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Guard the clean-room env contract in run_evals.py and run_triggers.py.
+
+The eval result is only honest if nothing from the host shell leaks into the
+subprocess. Both scripts carry their own build_case_env (they are
+deliberately self-contained); this test pins the contract on both copies:
+exactly PATH + fresh HOME + CLAUDE_CONFIG_DIR + auth-var-only-when-set +
+declared passthrough keys, nothing else.
+Run with: python3 -m pytest test_env_isolation.py
+(or plain `python3 test_env_isolation.py` for a lightweight self-check).
+"""
+import sys
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+import run_evals  # noqa: E402
+import run_triggers  # noqa: E402
+
+BUILDERS = [run_evals.build_case_env, run_triggers.build_case_env]
+
+HOST_ENV = {
+    "PATH": "/usr/bin:/bin",
+    "HOME": "/Users/host",
+    "ANTHROPIC_API_KEY": "sk-test-123",
+    "AWS_SECRET_ACCESS_KEY": "host-secret-must-not-leak",
+    "CLAUDE_CONFIG_DIR": "/Users/host/.claude",
+    "EXTRA_VAR": "extra",
+}
+
+HOME = Path("/tmp/eval-case/.home")
+
+
+def test_minimal_env_keys():
+    adapter = {"auth_env": "ANTHROPIC_API_KEY"}
+    for build in BUILDERS:
+        env = build(adapter, HOME, HOST_ENV)
+        assert set(env) == {"PATH", "HOME", "CLAUDE_CONFIG_DIR",
+                            "ANTHROPIC_API_KEY"}, (build.__module__, env)
+        assert env["PATH"] == HOST_ENV["PATH"]
+        assert env["HOME"] == str(HOME), "HOME must be the fresh case home"
+        assert env["CLAUDE_CONFIG_DIR"] == str(HOME / ".claude")
+        assert env["ANTHROPIC_API_KEY"] == "sk-test-123"
+        assert "AWS_SECRET_ACCESS_KEY" not in env, "host secrets leaked"
+
+
+def test_auth_var_absent_when_unset():
+    # Setting auth to "" breaks the runtime's OAuth fallback — the key must
+    # be absent, never empty.
+    adapter = {"auth_env": "ANTHROPIC_API_KEY"}
+    for host in ({}, {"ANTHROPIC_API_KEY": ""}):
+        for build in BUILDERS:
+            env = build(adapter, HOME, {"PATH": "/bin", **host})
+            assert "ANTHROPIC_API_KEY" not in env, (build.__module__, env)
+
+
+def test_no_adapter_still_minimal():
+    for build in BUILDERS:
+        env = build(None, HOME, HOST_ENV)
+        assert set(env) == {"PATH", "HOME", "CLAUDE_CONFIG_DIR"}, env
+
+
+def test_env_passthrough_only_declared_and_present():
+    adapter = {"auth_env": "ANTHROPIC_API_KEY",
+               "env_passthrough": ["EXTRA_VAR", "NOT_SET_ON_HOST"]}
+    for build in BUILDERS:
+        env = build(adapter, HOME, HOST_ENV)
+        assert env.get("EXTRA_VAR") == "extra"
+        assert "NOT_SET_ON_HOST" not in env
+        assert "AWS_SECRET_ACCESS_KEY" not in env
+
+
+if __name__ == "__main__":
+    test_minimal_env_keys()
+    test_auth_var_absent_when_unset()
+    test_no_adapter_still_minimal()
+    test_env_passthrough_only_declared_and_present()
+    print("ok: build_case_env contract holds in run_evals and run_triggers")
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Guard trigger detection in run_triggers.py.
+
+The stream-json init event lists every discovered skill by name, so any
+detection that substring-matches the whole transcript reports a 100% trigger
+rate. These tests pin the rule: only tool_use events (a Skill call naming the
+synthetic skill, or a Read inside its directory) count as a load, and
+substring-style load signals are rejected outright.
+Run with: python3 -m pytest test_trigger_detection.py
+(or plain `python3 test_trigger_detection.py` for a lightweight self-check).
+"""
+import json
+import sys
+from pathlib import Path
+
+SCRIPTS_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIR))
+
+from run_triggers import detect_load, validate_load_signal  # noqa: E402
+
+NAME = "my-skill-trig-abc12345"
+
+
+def line(obj) -> str:
+    return json.dumps(obj)
+
+
+def init_event() -> str:
+    # Claude Code's init event advertises every discovered skill.
+    return line({"type": "system", "subtype": "init",
+                 "tools": ["Skill", "Read", "Bash"],
+                 "slash_commands": [], "skills": [NAME, "other-skill"]})
+
+
+def assistant(content) -> str:
+    return line({"type": "assistant", "message": {"content": content}})
+
+
+def test_init_event_alone_is_not_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "text", "text": "I can't help with that."}]),
+        line({"type": "result", "usage": {}}),
+    ])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_text_mention_is_not_a_load():
+    transcript = assistant(
+        [{"type": "text", "text": f"There is a skill called {NAME} available."}])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_skill_tool_call_is_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Skill",
+                    "input": {"skill": NAME}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is True
+
+
+def test_read_of_synthetic_skill_md_is_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Read",
+                    "input": {"file_path":
+                              f"/tmp/stage/.claude/skills/{NAME}/SKILL.md"}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is True
+
+
+def test_unrelated_tool_calls_are_not_a_load():
+    transcript = "\n".join([
+        init_event(),
+        assistant([{"type": "tool_use", "name": "Read",
+                    "input": {"file_path": "/tmp/stage/notes.md"}}]),
+        assistant([{"type": "tool_use", "name": "Skill",
+                    "input": {"skill": "other-skill"}}]),
+        assistant([{"type": "tool_use", "name": "Bash",
+                    "input": {"command": f"echo {NAME}"}}]),
+    ])
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_custom_tool_names_from_load_signal():
+    sig = {"skill_tool": "InvokeSkill", "read_tool": "OpenFile"}
+    hit = assistant([{"type": "tool_use", "name": "InvokeSkill",
+                      "input": {"name": NAME}}])
+    miss = assistant([{"type": "tool_use", "name": "Skill",
+                       "input": {"skill": NAME}}])
+    assert detect_load(hit, sig, NAME) is True
+    assert detect_load(miss, sig, NAME) is False, \
+        "default tool name must not fire when the adapter renames it"
+
+
+def test_garbage_lines_do_not_crash():
+    transcript = "not json\n\n{\"type\": 42}\n[1,2,3]\n"
+    assert detect_load(transcript, {}, NAME) is False
+
+
+def test_string_load_signal_rejected():
+    for fn, args in ((validate_load_signal, ({"type": "string"},)),
+                     (detect_load, ("", {"type": "string"}, NAME))):
+        try:
+            fn(*args)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError(
+                f"{fn.__name__} accepted a substring load_signal")
+
+
+if __name__ == "__main__":
+    test_init_event_alone_is_not_a_load()
+    test_text_mention_is_not_a_load()
+    test_skill_tool_call_is_a_load()
+    test_read_of_synthetic_skill_md_is_a_load()
+    test_unrelated_tool_calls_are_not_a_load()
+    test_custom_tool_names_from_load_signal()
+    test_garbage_lines_do_not_crash()
+    test_string_load_signal_rejected()
+    print("ok: trigger detection counts tool calls only; substring rejected")