3d71c651fc
- testcases/*.md: declarative test definitions (send, expect_response, expect_state, expect_actions, action) - runtime_test.py: standalone runner + pytest integration via conftest.py - /tests route: web UI showing last run results from results.json - /api/tests: serves results JSON - Two initial testcases: counter_state (UI actions) and pub_conversation (multi-turn, language switch, tool use, memorizer state) - pub_conversation: 19/20 passed on first run - Fix nm-text vertical overflow in node metrics bar Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
46 lines
1.4 KiB
Python
46 lines
1.4 KiB
Python
"""Pytest configuration: collect testcases/*.md as test items."""
|
|
|
|
import pytest
|
|
from pathlib import Path
|
|
from runtime_test import parse_testcase, CogTestRunner
|
|
|
|
|
|
def pytest_collect_file(parent, file_path):
|
|
if file_path.suffix == ".md" and file_path.parent.name == "testcases":
|
|
return TestCaseFile.from_parent(parent, path=file_path)
|
|
|
|
|
|
class TestCaseFile(pytest.File):
|
|
def collect(self):
|
|
tc = parse_testcase(self.path)
|
|
yield TestCaseItem.from_parent(self, name=tc["name"], testcase=tc)
|
|
|
|
|
|
class TestCaseItem(pytest.Item):
|
|
def __init__(self, name, parent, testcase):
|
|
super().__init__(name, parent)
|
|
self.testcase = testcase
|
|
|
|
def runtest(self):
|
|
runner = CogTestRunner()
|
|
results = runner.run(self.testcase)
|
|
# Collect failures
|
|
failures = [r for r in results if r["status"] == "FAIL"]
|
|
if failures:
|
|
msg = "\n".join(f"Step {r['step']}: {r['check']} — {r['detail']}" for r in failures)
|
|
raise TestCaseFailure(msg, results)
|
|
|
|
def repr_failure(self, excinfo, style=None):
|
|
if isinstance(excinfo.value, TestCaseFailure):
|
|
return f"\n{excinfo.value.args[0]}"
|
|
return super().repr_failure(excinfo, style)
|
|
|
|
def reportinfo(self):
|
|
return self.path, 0, f"testcase: {self.name}"
|
|
|
|
|
|
class TestCaseFailure(Exception):
|
|
def __init__(self, msg, results):
|
|
super().__init__(msg)
|
|
self.results = results
|