qc.contract

ContractTestSuite

ContractTestSuite(
    loading_errors: list[tuple[DataStream, Exception]],
    exclude: Optional[list[DataStream]] = None,
)

Bases: Suite

Test suite for validating data stream loading.

Converts the output of DataStream.load_all() into a test suite that can report on loading errors and distinguish between true errors and expected/excluded issues.

Attributes:

Name	Type	Description
loading_errors		List of tuples containing the data stream and the exception that occurred.
exclude		Optional list of data streams to exclude from error reporting.

Examples:

from contraqctor.contract import Dataset
from contraqctor.qc.contract import ContractTestSuite
from contraqctor.qc.base import Runner

# Create and attempt to load a dataset
dataset = create_complex_dataset()
loading_errors = dataset.load_all(strict=False)

# Create test suite to analyze loading errors
suite = ContractTestSuite(
    loading_errors,
    exclude=[dataset["optional_stream"]]  # Exclude known problematic stream
)

# Run tests
runner = Runner().add_suite(suite)
results = runner.run_all_with_progress()

Initialize the contract test suite.

Parameters:

Name	Type	Description	Default
loading_errors	list[tuple[DataStream, Exception]]	List of tuples containing data streams and their loading errors.	required
exclude	Optional[list[DataStream]]	Optional list of data streams to exclude from error reporting. These will be reported as warnings instead of failures.	None

Source code in src/contraqctor/qc/contract.py

def __init__(
    self, loading_errors: list[tuple[DataStream, Exception]], exclude: t.Optional[list[DataStream]] = None
):
    """Initialize the contract test suite.

    Args:
        loading_errors: List of tuples containing data streams and their loading errors.
        exclude: Optional list of data streams to exclude from error reporting.
            These will be reported as warnings instead of failures.
    """
    self.loading_errors = loading_errors
    self.exclude = exclude if exclude is not None else []

description property

description: Optional[str]

Get the description of the test suite from its docstring.

Returns:

Type	Description
Optional[str]	Optional[str]: The docstring of the class, or None if not available.

name property

name: str

Get the name of the test suite.

Returns:

Name	Type	Description
str	str	The name of the test suite class.

test_has_errors_on_load

test_has_errors_on_load()

Check if any non-excluded data streams had loading errors.

Source code in src/contraqctor/qc/contract.py

def test_has_errors_on_load(self):
    """Check if any non-excluded data streams had loading errors."""
    errors = [(ds, err) for ds, err in self.loading_errors if ds not in self.exclude]
    if errors:
        str_errors = "\n".join([f"{ds.resolved_name}" for ds, _ in errors])
        return self.fail_test(
            None,
            f"The following DataStreams raised errors on load: \n {str_errors}",
            context={"errors": errors},
        )
    else:
        return self.pass_test(None, "All DataStreams loaded successfully")

test_has_excluded_as_warnings

test_has_excluded_as_warnings()

Check if any excluded data streams had loading errors and report as warnings.

Source code in src/contraqctor/qc/contract.py

def test_has_excluded_as_warnings(self):
    """Check if any excluded data streams had loading errors and report as warnings."""
    warnings = [(ds, err) for ds, err in self.loading_errors if ds in self.exclude]
    if warnings:
        return self.warn_test(
            None,
            f"Found {len(warnings)} DataStreams that raised ignored errors on load.",
            context={"warnings": warnings},
        )
    else:
        return self.pass_test(None, "No excluded DataStreams raised errors on load")

get_tests

get_tests() -> Generator[ITest, None, None]

Find all methods starting with 'test'.

Yields:

Name	Type	Description
ITest	ITest	Test methods found in the suite.

Source code in src/contraqctor/qc/base.py

def get_tests(self) -> t.Generator[ITest, None, None]:
    """Find all methods starting with 'test'.

    Yields:
        ITest: Test methods found in the suite.
    """
    for name, method in inspect.getmembers(self, predicate=inspect.ismethod):
        if name.startswith("test"):
            yield method

pass_test

pass_test() -> Result

pass_test(result: Any) -> Result

pass_test(result: Any, message: str) -> Result

pass_test(result: Any, *, context: Any) -> Result

pass_test(
    result: Any, message: str, *, context: Any
) -> Result

pass_test(
    result: Any = None,
    message: Optional[str] = None,
    *,
    context: Optional[Any] = None,
) -> Result

Create a passing test result.

Parameters:

Name	Type	Description	Default
result	Any	The value to include in the test result.	None
message	Optional[str]	Optional message describing why the test passed.	None
context	Optional[Any]	Optional contextual data for the test result.	None

Returns:

Name	Type	Description
Result	Result	A Result object with PASSED status.

Source code in src/contraqctor/qc/base.py

def pass_test(
    self, result: t.Any = None, message: t.Optional[str] = None, *, context: t.Optional[t.Any] = None
) -> Result:
    """Create a passing test result.

    Args:
        result: The value to include in the test result.
        message: Optional message describing why the test passed.
        context: Optional contextual data for the test result.

    Returns:
        Result: A Result object with PASSED status.
    """
    calling_func_name, description = self._get_caller_info()

    return Result(
        status=Status.PASSED,
        result=result,
        test_name=calling_func_name,
        suite_name=self.name,
        message=message,
        context=context,
        description=description,
    )

warn_test

warn_test() -> Result

warn_test(result: Any) -> Result

warn_test(result: Any, message: str) -> Result

warn_test(result: Any, *, context: Any) -> Result

warn_test(
    result: Any, message: str, *, context: Any
) -> Result

warn_test(
    result: Any = None,
    message: Optional[str] = None,
    *,
    context: Optional[Any] = None,
) -> Result

Create a warning test result.

Creates a result with WARNING status, or FAILED if warnings are elevated.

Parameters:

Name	Type	Description	Default
result	Any	The value to include in the test result.	None
message	Optional[str]	Optional message describing the warning.	None
context	Optional[Any]	Optional contextual data for the test result.	None

Returns:

Name	Type	Description
Result	Result	A Result object with WARNING or FAILED status.

Source code in src/contraqctor/qc/base.py

def warn_test(
    self, result: t.Any = None, message: t.Optional[str] = None, *, context: t.Optional[t.Any] = None
) -> Result:
    """Create a warning test result.

    Creates a result with WARNING status, or FAILED if warnings are elevated.

    Args:
        result: The value to include in the test result.
        message: Optional message describing the warning.
        context: Optional contextual data for the test result.

    Returns:
        Result: A Result object with WARNING or FAILED status.
    """
    calling_func_name, description = self._get_caller_info()

    return Result(
        status=Status.WARNING if not _elevate_warning.get() else Status.FAILED,
        result=result,
        test_name=calling_func_name,
        suite_name=self.name,
        message=message,
        context=context,
        description=description,
    )

fail_test

fail_test() -> Result

fail_test(result: Any) -> Result

fail_test(result: Any, message: str) -> Result

fail_test(
    result: Any, message: str, *, context: Any
) -> Result

fail_test(
    result: Optional[Any] = None,
    message: Optional[str] = None,
    *,
    context: Optional[Any] = None,
) -> Result

Create a failing test result.

Parameters:

Name	Type	Description	Default
result	Optional[Any]	The value to include in the test result.	None
message	Optional[str]	Optional message describing why the test failed.	None
context	Optional[Any]	Optional contextual data for the test result.	None

Returns:

Name	Type	Description
Result	Result	A Result object with FAILED status.

Source code in src/contraqctor/qc/base.py

def fail_test(
    self, result: t.Optional[t.Any] = None, message: t.Optional[str] = None, *, context: t.Optional[t.Any] = None
) -> Result:
    """Create a failing test result.

    Args:
        result: The value to include in the test result.
        message: Optional message describing why the test failed.
        context: Optional contextual data for the test result.

    Returns:
        Result: A Result object with FAILED status.
    """
    calling_func_name, description = self._get_caller_info()

    return Result(
        status=Status.FAILED,
        result=result,
        test_name=calling_func_name,
        suite_name=self.name,
        message=message,
        context=context,
        description=description,
    )

skip_test

skip_test() -> Result

skip_test(message: str) -> Result

skip_test(message: str, *, context: Any) -> Result

skip_test(
    message: Optional[str] = None,
    *,
    context: Optional[Any] = None,
) -> Result

Create a skipped test result.

Creates a result with SKIPPED status, or FAILED if skips are elevated.

Parameters:

Name	Type	Description	Default
message	Optional[str]	Optional message explaining why the test was skipped.	None
context	Optional[Any]	Optional contextual data for the test result.	None

Returns:

Name	Type	Description
Result	Result	A Result object with SKIPPED or FAILED status.

Source code in src/contraqctor/qc/base.py

def skip_test(self, message: t.Optional[str] = None, *, context: t.Optional[t.Any] = None) -> Result:
    """Create a skipped test result.

    Creates a result with SKIPPED status, or FAILED if skips are elevated.

    Args:
        message: Optional message explaining why the test was skipped.
        context: Optional contextual data for the test result.

    Returns:
        Result: A Result object with SKIPPED or FAILED status.
    """
    calling_func_name, description = self._get_caller_info()
    return Result(
        status=Status.SKIPPED if not _elevate_skippable.get() else Status.FAILED,
        result=None,
        test_name=calling_func_name,
        suite_name=self.name,
        message=message,
        context=context,
        description=description,
    )

setup

setup() -> None

Run before each test method.

This method can be overridden by subclasses to implement setup logic that runs before each test.

Source code in src/contraqctor/qc/base.py

def setup(self) -> None:
    """Run before each test method.

    This method can be overridden by subclasses to implement
    setup logic that runs before each test.
    """
    pass

teardown

teardown() -> None

Run after each test method.

This method can be overridden by subclasses to implement teardown logic that runs after each test.

Source code in src/contraqctor/qc/base.py

def teardown(self) -> None:
    """Run after each test method.

    This method can be overridden by subclasses to implement
    teardown logic that runs after each test.
    """
    pass

run_test

run_test(
    test_method: ITest,
) -> Generator[Result, None, None]

Run a single test method and yield its results.

Handles setup, test execution, result processing, and teardown.

Parameters:

Name	Type	Description	Default
test_method	ITest	The test method to run.	required

Yields:

Name	Type	Description
Result	Result	Result objects produced by the test method.

Source code in src/contraqctor/qc/base.py

def run_test(self, test_method: ITest) -> t.Generator[Result, None, None]:
    """Run a single test method and yield its results.

    Handles setup, test execution, result processing, and teardown.

    Args:
        test_method: The test method to run.

    Yields:
        Result: Result objects produced by the test method.
    """
    test_name = test_method.__name__
    suite_name = self.name
    test_description = getattr(test_method, "__doc__", None)

    try:
        self.setup()
        result = test_method()
        if inspect.isgenerator(result):
            for sub_result in result:
                yield self._process_test_result(sub_result, test_method, test_name, test_description)
        else:
            yield self._process_test_result(result, test_method, test_name, test_description)
    except Exception as e:
        tb = traceback.format_exc()
        yield Result(
            status=Status.ERROR,
            result=None,
            test_name=test_name,
            suite_name=suite_name,
            description=test_description,
            message=f"Error during test execution: {str(e)}",
            exception=e,
            traceback=tb,
            test_reference=test_method,
            suite_reference=self,
        )
    finally:
        self.teardown()

run_all

run_all() -> Generator[Result, None, None]

Run all test methods in the suite.

Finds all test methods and runs them in sequence.

Yields:

Name	Type	Description
Result	Result	Result objects produced by all test methods.

Source code in src/contraqctor/qc/base.py

def run_all(self) -> t.Generator[Result, None, None]:
    """Run all test methods in the suite.

    Finds all test methods and runs them in sequence.

    Yields:
        Result: Result objects produced by all test methods.
    """
    for test in self.get_tests():
        yield from self.run_test(test)