Skip to content

Classes

spark_expectations.sinks.utils.collect_statistics.SparkExpectationsCollectStatistics(_context: SparkExpectationsContext, _writer: SparkExpectationsWriter) dataclass

This class implements logging statistics on success and failure

Functions

collect_stats_on_success_failure() -> Any

The function implements decorator to log statistics on success and failure Returns: Any: function

Source code in spark_expectations/sinks/utils/collect_statistics.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def collect_stats_on_success_failure(self) -> Any:
    """
    The function implements decorator to log statistics on success and failure
    Returns:
        Any: function
    """

    def decorator(func: Any) -> Any:
        def wrapper(*args: List, **kwargs: Dict) -> DataFrame:
            try:
                self._context.set_dq_start_time()

                result = func(*args, **kwargs)

                self._context.set_dq_run_status("Passed")
                self._context.set_dq_end_time()

                self._writer.write_error_stats()
            except Exception as e:
                self._context.set_dq_run_status("Failed")
                self._context.set_end_time_when_dq_job_fails()
                self._context.set_dq_end_time()

                self._writer.write_error_stats()
                raise SparkExpectationsMiscException(e)
            return result

        return wrapper

    return decorator