- Introduced a new HOOK_VALIDATION_REPORT.md to document the results of comprehensive testing for code quality hooks. - Implemented integration tests for core blocking functionality, ensuring that forbidden patterns (e.g., typing.Any, type: ignore, old typing patterns) are properly blocked. - Added tests for command line execution and enforcement modes to validate hook behavior in various scenarios. - Enhanced detection functions for typing.Any usage, type: ignore, and old typing patterns with comprehensive test cases. - Improved error handling and messaging in the hook system to provide clearer feedback on violations. - Established a structured approach to testing, ensuring all critical hooks are validated and functioning correctly.
169 lines
5.7 KiB
Python
169 lines
5.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Core hook validation tests - MUST ALL PASS
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add hooks to path
|
|
sys.path.insert(0, str(Path(__file__).parent / "hooks"))
|
|
|
|
from code_quality_guard import pretooluse_hook, QualityConfig
|
|
|
|
|
|
def test_core_blocking():
|
|
"""Test the core blocking functionality that MUST work."""
|
|
config = QualityConfig.from_env()
|
|
config.enforcement_mode = "strict"
|
|
|
|
tests_passed = 0
|
|
tests_failed = 0
|
|
|
|
# Test 1: Any usage MUST be blocked
|
|
print("🧪 Test 1: Any usage blocking")
|
|
hook_data = {
|
|
"tool_name": "Write",
|
|
"tool_input": {
|
|
"file_path": "/src/production.py",
|
|
"content": "from typing import Any\ndef bad(x: Any) -> Any: return x"
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "deny" and "typing.Any usage" in result.get("reason", ""):
|
|
print("✅ PASS: Any usage properly blocked")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: Any usage not blocked. Decision: {result['permissionDecision']}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in Any test: {e}")
|
|
tests_failed += 1
|
|
|
|
# Test 2: type: ignore MUST be blocked
|
|
print("\n🧪 Test 2: type: ignore blocking")
|
|
hook_data = {
|
|
"tool_name": "Write",
|
|
"tool_input": {
|
|
"file_path": "/src/production.py",
|
|
"content": "def bad():\n x = call() # type: ignore\n return x"
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "deny" and "type: ignore" in result.get("reason", ""):
|
|
print("✅ PASS: type: ignore properly blocked")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: type: ignore not blocked. Decision: {result['permissionDecision']}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in type: ignore test: {e}")
|
|
tests_failed += 1
|
|
|
|
# Test 3: Old typing patterns MUST be blocked
|
|
print("\n🧪 Test 3: Old typing patterns blocking")
|
|
hook_data = {
|
|
"tool_name": "Write",
|
|
"tool_input": {
|
|
"file_path": "/src/production.py",
|
|
"content": "from typing import Union, Optional\ndef bad(x: Union[str, int]) -> Optional[str]: return None"
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "deny" and "Old typing pattern" in result.get("reason", ""):
|
|
print("✅ PASS: Old typing patterns properly blocked")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: Old typing patterns not blocked. Decision: {result['permissionDecision']}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in old typing test: {e}")
|
|
tests_failed += 1
|
|
|
|
# Test 4: Good code MUST be allowed
|
|
print("\n🧪 Test 4: Good code allowed")
|
|
hook_data = {
|
|
"tool_name": "Write",
|
|
"tool_input": {
|
|
"file_path": "/src/production.py",
|
|
"content": "def good(x: str | int) -> str | None:\n return str(x) if x else None"
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "allow":
|
|
print("✅ PASS: Good code properly allowed")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: Good code blocked. Decision: {result['permissionDecision']}")
|
|
print(f" Reason: {result.get('reason', 'No reason')}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in good code test: {e}")
|
|
tests_failed += 1
|
|
|
|
# Test 5: Edit tool MUST also block
|
|
print("\n🧪 Test 5: Edit tool blocking")
|
|
hook_data = {
|
|
"tool_name": "Edit",
|
|
"tool_input": {
|
|
"file_path": "/src/production.py",
|
|
"old_string": "def old():",
|
|
"new_string": "def new(x: Any) -> Any:"
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "deny" and "typing.Any usage" in result.get("reason", ""):
|
|
print("✅ PASS: Edit tool properly blocked")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: Edit tool not blocked. Decision: {result['permissionDecision']}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in Edit tool test: {e}")
|
|
tests_failed += 1
|
|
|
|
# Test 6: Non-Python files MUST be allowed
|
|
print("\n🧪 Test 6: Non-Python files allowed")
|
|
hook_data = {
|
|
"tool_name": "Write",
|
|
"tool_input": {
|
|
"file_path": "/src/config.json",
|
|
"content": '{"type": "Any", "ignore": true}'
|
|
}
|
|
}
|
|
|
|
try:
|
|
result = pretooluse_hook(hook_data, config)
|
|
if result["permissionDecision"] == "allow":
|
|
print("✅ PASS: Non-Python files properly allowed")
|
|
tests_passed += 1
|
|
else:
|
|
print(f"❌ FAIL: Non-Python file blocked. Decision: {result['permissionDecision']}")
|
|
tests_failed += 1
|
|
except Exception as e:
|
|
print(f"❌ FAIL: Exception in non-Python test: {e}")
|
|
tests_failed += 1
|
|
|
|
print(f"\n📊 Results: {tests_passed} passed, {tests_failed} failed")
|
|
|
|
if tests_failed == 0:
|
|
print("🎉 ALL CORE TESTS PASSED! Hooks are working correctly.")
|
|
return True
|
|
else:
|
|
print(f"💥 {tests_failed} CRITICAL TESTS FAILED! Hooks are broken.")
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print("🚀 Running core hook validation tests...\n")
|
|
success = test_core_blocking()
|
|
sys.exit(0 if success else 1) |