HimanshuJ16
48bcc05379
- Changed minimum Python version from 3.7+ or 3.9+ to 3.10+ in various skill documentation files to reflect the use of features introduced in Python 3.10. - Updated prerequisites in skills related to structured logging, futures contract construction, systemd supervision, tail correlation, risk hedging, and more. - Ensured consistency in Python version claims across all relevant skills to maintain compatibility with the latest library features. - Added validation tests to enforce the new Python floor requirement and prevent claims below 3.10 in skill documentation.
289 lines
13 KiB
Python
289 lines
13 KiB
Python
"""
|
|
Behavioral unit tests for tools/validate_skills.py.
|
|
"""
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
|
|
from tools.validate_skills import (
|
|
validate_skill_dir,
|
|
parse_frontmatter,
|
|
CANONICAL_TEST_CMD,
|
|
DESCRIPTION_MAX,
|
|
REQUIRED_FRONTMATTER_FIELDS,
|
|
REQUIRED_METADATA_FIELDS,
|
|
REQUIRED_SECTIONS,
|
|
REQUIRED_SUBDIRS,
|
|
REQUIRED_FILES,
|
|
)
|
|
|
|
|
|
def frontmatter(name, **overrides):
|
|
"""A spec-conformant frontmatter block; overrides replace metadata or top-level keys."""
|
|
top = {
|
|
"name": name,
|
|
"description": "Use when a test needs a spec-conformant skill fixture to validate against",
|
|
"license": "Apache-2.0",
|
|
}
|
|
meta = {
|
|
"domain": "algorithmic-trading",
|
|
"subdomain": "risk-management",
|
|
"tags": "test, risk",
|
|
"brokers_frameworks": "Universal",
|
|
"version": '"1.0.0"',
|
|
"author": "algo-trading-skills-contributors",
|
|
}
|
|
for k, v in overrides.items():
|
|
if k in meta:
|
|
meta[k] = v
|
|
else:
|
|
top[k] = v
|
|
lines = ["---"]
|
|
for k, v in top.items():
|
|
if v is not None:
|
|
lines.append(f"{k}: {v}")
|
|
lines.append("metadata:")
|
|
for k, v in meta.items():
|
|
if v is not None:
|
|
lines.append(f" {k}: {v}")
|
|
lines.append("---")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
class TestSkillValidation(unittest.TestCase):
|
|
def setUp(self):
|
|
self.test_dir = tempfile.mkdtemp()
|
|
|
|
def tearDown(self):
|
|
shutil.rmtree(self.test_dir)
|
|
|
|
@staticmethod
|
|
def _body(name):
|
|
"""A body carrying every required section, with a runnable Verification."""
|
|
sections = []
|
|
for sec in REQUIRED_SECTIONS:
|
|
if sec == "Verification":
|
|
cmd = CANONICAL_TEST_CMD.format(name=name)
|
|
sections.append(f"## {sec}\n- Run `{cmd}`.")
|
|
else:
|
|
sections.append(f"## {sec}\nContent for {sec}")
|
|
return "\n".join(sections)
|
|
|
|
def _create_valid_skill(self, name="test-skill", fm=None):
|
|
skill_dir = os.path.join(self.test_dir, name)
|
|
os.makedirs(skill_dir)
|
|
for sd in REQUIRED_SUBDIRS:
|
|
os.makedirs(os.path.join(skill_dir, sd))
|
|
for rel in REQUIRED_FILES:
|
|
with open(os.path.join(skill_dir, rel), "w", encoding="utf-8") as f:
|
|
f.write("content\n")
|
|
with open(os.path.join(skill_dir, "scripts", "helper.py"), "w", encoding="utf-8") as f:
|
|
f.write("VALUE = 1\n")
|
|
with open(os.path.join(skill_dir, "scripts", "test_helper.py"), "w", encoding="utf-8") as f:
|
|
f.write("import helper\n")
|
|
with open(os.path.join(skill_dir, "SKILL.md"), "w", encoding="utf-8") as f:
|
|
f.write((fm or frontmatter(name)) + self._body(name))
|
|
return skill_dir
|
|
|
|
def _errors(self, name, **overrides):
|
|
skill_dir = self._create_valid_skill(name, frontmatter(name, **overrides))
|
|
return validate_skill_dir(skill_dir, {name})
|
|
|
|
def test_valid_skill_passes(self):
|
|
skill_dir = self._create_valid_skill("valid-skill")
|
|
self.assertEqual(validate_skill_dir(skill_dir, {"valid-skill"}), [])
|
|
|
|
def test_parse_frontmatter_handles_crlf(self):
|
|
fm, body = parse_frontmatter(frontmatter("x-y-z").replace("\n", "\r\n") + "body")
|
|
self.assertEqual(fm["name"], "x-y-z")
|
|
self.assertEqual(fm["metadata"]["subdomain"], "risk-management")
|
|
self.assertEqual(body, "body")
|
|
|
|
def test_missing_skill_md_fails(self):
|
|
skill_dir = os.path.join(self.test_dir, "missing-md")
|
|
os.makedirs(skill_dir)
|
|
errors = validate_skill_dir(skill_dir, set())
|
|
self.assertTrue(any("missing SKILL.md" in e for e in errors))
|
|
|
|
def test_missing_frontmatter_fails(self):
|
|
skill_dir = os.path.join(self.test_dir, "no-fm")
|
|
os.makedirs(skill_dir)
|
|
with open(os.path.join(skill_dir, "SKILL.md"), "w", encoding="utf-8") as f:
|
|
f.write("No frontmatter block here")
|
|
errors = validate_skill_dir(skill_dir, set())
|
|
self.assertTrue(any("missing YAML frontmatter" in e for e in errors))
|
|
|
|
def test_missing_required_field_fails(self):
|
|
errors = self._errors("field-missing", author=None)
|
|
self.assertTrue(any("metadata missing required field 'author'" in e for e in errors))
|
|
self.assertIn("author", REQUIRED_METADATA_FIELDS)
|
|
self.assertIn("metadata", REQUIRED_FRONTMATTER_FIELDS)
|
|
|
|
def test_repo_field_at_top_level_fails(self):
|
|
"""agentskills.io allows only name/description/license/compatibility/allowed-tools/metadata."""
|
|
errors = self._errors("top-level-tags", tags2=None) # no-op override
|
|
self.assertEqual(errors, [])
|
|
skill_dir = os.path.join(self.test_dir, "top-level-tags")
|
|
path = os.path.join(skill_dir, "SKILL.md")
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text.replace("license: Apache-2.0\n", "license: Apache-2.0\ntags: [a, b]\n"))
|
|
errors = validate_skill_dir(skill_dir, {"top-level-tags"})
|
|
self.assertTrue(any("unexpected top-level frontmatter field" in e for e in errors))
|
|
|
|
def test_unknown_subdomain_fails(self):
|
|
errors = self._errors("bad-subdomain", subdomain="Quantitative Wizardry")
|
|
self.assertTrue(any("not one of the 16 domains" in e for e in errors))
|
|
|
|
def test_wrong_license_or_author_fails(self):
|
|
errors = self._errors("mit-skill", license="MIT")
|
|
self.assertTrue(any("license must be 'Apache-2.0'" in e for e in errors))
|
|
errors = self._errors("odd-author", author="Quant Team")
|
|
self.assertTrue(any("metadata.author must be" in e for e in errors))
|
|
|
|
def test_description_must_be_a_trigger_and_short(self):
|
|
errors = self._errors("noun-desc", description="Institutional compliance skill for reg reporting")
|
|
self.assertTrue(any("must start with 'Use when" in e for e in errors))
|
|
self.assertTrue(any("marketing word" in e for e in errors))
|
|
errors = self._errors("long-desc", description="Use when " + "x" * DESCRIPTION_MAX)
|
|
self.assertTrue(any("maximum is" in e for e in errors))
|
|
|
|
def test_unquoted_float_version_fails(self):
|
|
errors = self._errors("float-version", version="1.10")
|
|
self.assertTrue(any("must be a string" in e for e in errors))
|
|
|
|
def test_name_length_and_shape(self):
|
|
long_name = "a" * 65
|
|
skill_dir = self._create_valid_skill(long_name, frontmatter(long_name))
|
|
errors = validate_skill_dir(skill_dir, {long_name})
|
|
self.assertTrue(any("at most 64 chars" in e for e in errors))
|
|
|
|
def test_section_heading_must_be_level_two(self):
|
|
skill_dir = self._create_valid_skill("h3-section")
|
|
path = os.path.join(skill_dir, "SKILL.md")
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text.replace("## Common Pitfalls", "### Common Pitfalls"))
|
|
errors = validate_skill_dir(skill_dir, {"h3-section"})
|
|
self.assertTrue(any("missing required section '## Common Pitfalls'" in e for e in errors))
|
|
|
|
def test_broken_related_skill_reference_fails(self):
|
|
skill_dir = self._create_valid_skill("has-broken-ref")
|
|
path = os.path.join(skill_dir, "SKILL.md")
|
|
with open(path, "a", encoding="utf-8") as f:
|
|
f.write("\n- `no-such-skill-here`\n")
|
|
errors = validate_skill_dir(skill_dir, {"has-broken-ref"})
|
|
self.assertTrue(any("broken Related Skills reference `no-such-skill-here`" in e for e in errors))
|
|
|
|
def test_missing_subdirectory_fails(self):
|
|
skill_dir = self._create_valid_skill("sub-missing")
|
|
shutil.rmtree(os.path.join(skill_dir, "references"))
|
|
errors = validate_skill_dir(skill_dir, {"sub-missing"})
|
|
self.assertTrue(any("missing required subdirectory 'references/'" in e for e in errors))
|
|
|
|
def test_missing_named_file_fails(self):
|
|
skill_dir = self._create_valid_skill("no-checklist")
|
|
os.remove(os.path.join(skill_dir, "assets", "checklist.md"))
|
|
with open(os.path.join(skill_dir, "assets", "other.md"), "w", encoding="utf-8") as f:
|
|
f.write("x\n")
|
|
errors = validate_skill_dir(skill_dir, {"no-checklist"})
|
|
self.assertTrue(any("missing required file 'assets/checklist.md'" in e for e in errors))
|
|
|
|
def test_helper_named_like_a_test_fails(self):
|
|
"""run_all_tests.py globs test_*.py, so a helper matching it is collected as a suite."""
|
|
skill_dir = self._create_valid_skill("helper-named-test")
|
|
scripts = os.path.join(skill_dir, "scripts")
|
|
os.rename(os.path.join(scripts, "helper.py"), os.path.join(scripts, "test_helper_module.py"))
|
|
errors = validate_skill_dir(skill_dir, {"helper-named-test"})
|
|
self.assertTrue(any("no helper module" in e for e in errors))
|
|
|
|
def test_missing_test_suite_fails(self):
|
|
skill_dir = self._create_valid_skill("no-suite")
|
|
os.remove(os.path.join(skill_dir, "scripts", "test_helper.py"))
|
|
errors = validate_skill_dir(skill_dir, {"no-suite"})
|
|
self.assertTrue(any("no test_*.py suite" in e for e in errors))
|
|
|
|
def test_verification_without_test_command_fails(self):
|
|
skill_dir = self._create_valid_skill("no-cmd")
|
|
path = os.path.join(skill_dir, "SKILL.md")
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
text = text.replace(f"- Run `{CANONICAL_TEST_CMD.format(name='no-cmd')}`.", "- Eyeball the output.")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
errors = validate_skill_dir(skill_dir, {"no-cmd"})
|
|
self.assertTrue(any("never shows how to run the tests" in e for e in errors))
|
|
|
|
def test_cwd_relative_test_command_fails(self):
|
|
"""`python scripts/test_x.py` does not resolve from the repository root."""
|
|
skill_dir = self._create_valid_skill("relative-cmd")
|
|
path = os.path.join(skill_dir, "assets", "checklist.md")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("- [ ] Run `python scripts/test_helper.py`.\n")
|
|
errors = validate_skill_dir(skill_dir, {"relative-cmd"})
|
|
self.assertTrue(any("does not run from the repository root" in e for e in errors))
|
|
|
|
def test_cd_into_skill_before_command_fails(self):
|
|
"""`cd skills/x/scripts` on the line before the repo-root command breaks it."""
|
|
skill_dir = self._create_valid_skill("cd-cmd")
|
|
path = os.path.join(skill_dir, "SKILL.md")
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
cmd = CANONICAL_TEST_CMD.format(name="cd-cmd")
|
|
text = text.replace(f"- Run `{cmd}`.",
|
|
f"```bash\ncd skills/cd-cmd/scripts\n{cmd}\n```")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
errors = validate_skill_dir(skill_dir, {"cd-cmd"})
|
|
self.assertTrue(any("drop the cd" in e for e in errors), errors)
|
|
|
|
def test_python_floor_below_library_floor_fails(self):
|
|
skill_dir = self._create_valid_skill("old-python")
|
|
path = os.path.join(skill_dir, "references", "workflows.md")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("Requires Python 3.9+ for `math.nextafter`.\n")
|
|
errors = validate_skill_dir(skill_dir, {"old-python"})
|
|
self.assertTrue(any("below the library floor" in e for e in errors), errors)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("Requires Python 3.10+ (standard library only).\n")
|
|
self.assertEqual(validate_skill_dir(skill_dir, {"old-python"}), [])
|
|
|
|
def test_python3_and_pytest_forms_are_caught(self):
|
|
skill_dir = self._create_valid_skill("python3-cmd")
|
|
path = os.path.join(skill_dir, "references", "workflows.md")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("Run `python3 -m unittest discover -s scripts`.\n")
|
|
errors = validate_skill_dir(skill_dir, {"python3-cmd"})
|
|
self.assertTrue(any("does not run from the repository root" in e for e in errors))
|
|
|
|
def test_whole_repo_runner_is_allowed(self):
|
|
skill_dir = self._create_valid_skill("repo-runner")
|
|
path = os.path.join(skill_dir, "assets", "checklist.md")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("- [ ] Run `python tools/run_all_tests.py`.\n")
|
|
self.assertEqual(validate_skill_dir(skill_dir, {"repo-runner"}), [])
|
|
|
|
def test_verbose_flag_is_allowed(self):
|
|
skill_dir = self._create_valid_skill("verbose-cmd")
|
|
path = os.path.join(skill_dir, "references", "workflows.md")
|
|
cmd = CANONICAL_TEST_CMD.format(name="verbose-cmd")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(f"Run `{cmd} -v` for per-test output.\n")
|
|
self.assertEqual(validate_skill_dir(skill_dir, {"verbose-cmd"}), [])
|
|
|
|
def test_line_wrapped_command_is_allowed(self):
|
|
"""Markdown wraps long commands; the check compares on collapsed whitespace."""
|
|
skill_dir = self._create_valid_skill("wrapped-cmd")
|
|
path = os.path.join(skill_dir, "assets", "checklist.md")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write("- [ ] Run\n `python -m unittest discover -s\n"
|
|
" skills/wrapped-cmd/scripts`.\n")
|
|
self.assertEqual(validate_skill_dir(skill_dir, {"wrapped-cmd"}), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|