"""Time limit definitions and validation for test categories.
Test sizes are DEFINITIONS, not configurable options. This follows Google's
"Software Engineering at Google" philosophy where test sizes have fixed
meanings:
- Small tests (< 1s): Fast unit tests without external dependencies
- Medium tests (< 5 min): Integration tests with local services
- Large tests (< 15 min): Full system/E2E tests
- XLarge tests (< 15 min): Extended tests with same limits as large
If a test exceeds its category's time limit, the correct action is to
RECATEGORIZE the test to a larger size, not extend the limit.
"""
from __future__ import annotations
from typing import Annotated
from pydantic import (
BaseModel,
ConfigDict,
Field,
)
from pytest_test_categories.errors import (
ERROR_CODES,
format_error_message,
)
from pytest_test_categories.types import TestSize
__all__ = [
'LARGE_LIMIT',
'MEDIUM_LIMIT',
'SMALL_LIMIT',
'TIME_LIMITS',
'XLARGE_LIMIT',
'PerformanceBaselineViolationError',
'TimeLimit',
'TimingViolationError',
'get_limit',
'validate',
'validate_with_baseline',
]
[docs]
class TimingViolationError(Exception):
"""Exception raised when a test exceeds its time limit.
This exception is raised when a test's execution time exceeds the
configured time limit for its size category.
The error message includes:
- Error code [TC006]
- Test identification (nodeid, size category)
- Timing details (limit vs actual duration)
- Why timing limits matter
- Remediation suggestions
- Documentation link
Attributes:
test_size: The test's size category.
test_nodeid: The pytest node ID of the failing test.
limit: The time limit in seconds.
actual: The actual test duration in seconds.
Example:
>>> raise TimingViolationError(
... test_size=TestSize.SMALL,
... test_nodeid='tests/test_slow.py::test_compute',
... limit=1.0,
... actual=2.5
... )
"""
def __init__(
self,
test_size: TestSize,
test_nodeid: str,
limit: float,
actual: float,
) -> None:
"""Initialize a timing violation error.
Args:
test_size: The test's size category.
test_nodeid: The pytest node ID of the failing test.
limit: The time limit in seconds.
actual: The actual test duration in seconds.
"""
[docs]
self.test_size = test_size
[docs]
self.test_nodeid = test_nodeid
remediation = self._get_remediation(test_size)
what_happened = f'{test_size.name} test exceeded time limit of {limit:.1f} seconds (took {actual:.1f} seconds)'
message = format_error_message(
error_code=ERROR_CODES['timing_violation'],
what_happened=what_happened,
remediation=remediation,
test_nodeid=test_nodeid,
test_size=test_size.name,
)
super().__init__(message)
@staticmethod
def _get_remediation(test_size: TestSize) -> list[str]:
"""Get remediation suggestions based on test size.
Args:
test_size: The test's size category.
Returns:
List of remediation suggestions.
"""
next_size = {
TestSize.SMALL: '@pytest.mark.medium',
TestSize.MEDIUM: '@pytest.mark.large',
TestSize.LARGE: '@pytest.mark.xlarge',
TestSize.XLARGE: None,
}
suggestions = [
'Optimize the test to run faster (reduce setup, use fixtures)',
'Mock slow dependencies (network, filesystem, database)',
'Split the test into smaller, focused tests',
]
next_marker = next_size.get(test_size)
if next_marker:
suggestions.append(f'Change test category to {next_marker} (if more time is genuinely needed)')
else:
suggestions.append('Review if this test is doing too much work')
return suggestions
[docs]
class TimeLimit(BaseModel):
"""Fixed time limit for a test size category.
This is an immutable value object representing a test size's time limit.
Time limits are fixed definitions based on Google's test size standards,
not configurable options.
"""
[docs]
limit: Annotated[float, Field(gt=0)] # Time limit in seconds must be positive
[docs]
model_config = ConfigDict(frozen=True)
# Fixed time limits matching Google's test size definitions
# These are DEFINITIONS, not defaults that can be overridden
[docs]
SMALL_LIMIT = TimeLimit(limit=1.0)
[docs]
MEDIUM_LIMIT = TimeLimit(limit=300.0)
[docs]
LARGE_LIMIT = TimeLimit(limit=900.0)
[docs]
XLARGE_LIMIT = TimeLimit(limit=900.0)
# Mapping of test sizes to their fixed limits
[docs]
TIME_LIMITS = {
TestSize.SMALL: SMALL_LIMIT,
TestSize.MEDIUM: MEDIUM_LIMIT,
TestSize.LARGE: LARGE_LIMIT,
TestSize.XLARGE: XLARGE_LIMIT,
}
[docs]
def get_limit(size: TestSize) -> TimeLimit:
"""Get the fixed time limit for a test size.
Args:
size: The test size category.
Returns:
The TimeLimit for the given test size.
Example:
>>> get_limit(TestSize.SMALL).limit
1.0
>>> get_limit(TestSize.MEDIUM).limit
300.0
"""
return TIME_LIMITS[size]
[docs]
def validate(
size: TestSize,
duration: float,
test_nodeid: str = '',
) -> None:
"""Validate a test's duration against its size's fixed time limit.
Test sizes have fixed time limits that are not configurable. If a test
exceeds its limit, the correct action is to recategorize the test to
a larger size, not extend the limit.
Args:
size: The test size category.
duration: The actual test duration in seconds.
test_nodeid: Optional pytest node ID for enhanced error messages.
Raises:
TimingViolationError: If the test exceeds its time limit.
Example:
>>> validate(TestSize.SMALL, 0.5) # Passes (0.5s < 1s limit)
>>> validate(TestSize.SMALL, 2.0) # Raises TimingViolationError
>>> validate(TestSize.SMALL, 2.0, test_nodeid='tests/test_slow.py::test_compute')
"""
limit = get_limit(size).limit
if duration > limit:
raise TimingViolationError(
test_size=size,
test_nodeid=test_nodeid,
limit=limit,
actual=duration,
)
[docs]
def validate_with_baseline(
size: TestSize,
duration: float,
baseline: float | None,
test_nodeid: str = '',
) -> None:
"""Validate a test's duration against a custom baseline or category limit.
When a custom baseline is provided, the test must complete within that
stricter limit. If the baseline is exceeded, a PerformanceBaselineViolationError
is raised. If no baseline is provided, falls back to standard category limit
validation.
The baseline must be less than or equal to the category's time limit.
This ensures that custom baselines are always stricter than the default.
Args:
size: The test size category.
duration: The actual test duration in seconds.
baseline: Optional custom performance baseline in seconds.
If None, uses the category's default time limit.
test_nodeid: Optional pytest node ID for enhanced error messages.
Raises:
PerformanceBaselineViolationError: If duration exceeds the custom baseline.
TimingViolationError: If duration exceeds category limit (when no baseline).
ValueError: If baseline exceeds the category's time limit.
Example:
>>> # Test with custom baseline
>>> validate_with_baseline(TestSize.SMALL, 0.05, baseline=0.1) # OK
>>> validate_with_baseline(TestSize.SMALL, 0.15, baseline=0.1) # Raises PerformanceBaselineViolationError
>>> # Test without baseline (uses category limit)
>>> validate_with_baseline(TestSize.SMALL, 0.5, baseline=None) # OK
>>> validate_with_baseline(TestSize.SMALL, 1.5, baseline=None) # Raises TimingViolationError
"""
category_limit = get_limit(size).limit
# If no baseline, fall back to standard validation
if baseline is None:
validate(size, duration, test_nodeid)
return
# Validate that baseline doesn't exceed category limit
if baseline > category_limit:
msg = f'baseline ({baseline}s) cannot exceed category limit ({category_limit}s)'
raise ValueError(msg)
# Check against custom baseline
if duration > baseline:
raise PerformanceBaselineViolationError(
test_size=size,
test_nodeid=test_nodeid,
baseline_limit=baseline,
category_limit=category_limit,
actual=duration,
)