Skip to content

Classes

spark_expectations.sinks.utils.collect_statistics.SparkExpectationsCollectStatistics dataclass

This class implements logging statistics on success and failure

Functions

collect_stats_on_success_failure() -> Any

The function implements decorator to log statistics on success and failure Returns: Any: function

Source code in spark_expectations/sinks/utils/collect_statistics.py
def collect_stats_on_success_failure(self) -> Any:
    """
    The function implements decorator to log statistics on success and failure
    Returns:
        Any: function
    """

    def decorator(func: Any) -> Any:
        def wrapper(*args: List, **kwargs: Dict) -> DataFrame:
            try:
                self._context.set_dq_start_time()

                result = func(*args, **kwargs)

                self._context.set_dq_run_status("Passed")
                self._context.set_dq_end_time()

                self._writer.write_error_stats()
            except Exception as e:
                self._context.set_dq_run_status("Failed")
                self._context.set_end_time_when_dq_job_fails()
                self._context.set_dq_end_time()

                self._writer.write_error_stats()
                raise SparkExpectationsMiscException(e)
            return result

        return wrapper

    return decorator