Skip to content

Attributes

Classes

Functions

spark_expectations.sinks.get_sink_hook() -> pluggy.PluginManager cached

function provides pluggy hook manager to write data into delta and nsp

Returns:

Name Type Description
PluginManager pluggy.PluginManager

pluggy Manager object
Source code in spark_expectations/sinks/__init__.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
@functools.lru_cache
def get_sink_hook() -> pluggy.PluginManager:
    """
    function provides pluggy hook manager to write data into delta and nsp
    Returns:
        PluginManager: pluggy Manager object

    """
    pm = pluggy.PluginManager(SPARK_EXPECTATIONS_WRITER_PLUGIN)
    pm.add_hookspecs(SparkExpectationsSinkWriter)
    pm.register(
        SparkExpectationsDeltaWritePluginImpl(), "spark_expectations_delta_write"
    )
    pm.register(
        SparkExpectationsKafkaWritePluginImpl(), "spark_expectations_kafka_write"
    )
    for name, plugin_instance in pm.list_name_plugin():
        _log.info(
            "Loaded plugin with name: %s and class: %s",
            name,
            plugin_instance.__class__.__name__,
        )
    return pm