#80: Refactoring project structure according to feedback. Fixed impor…

…ts and black-linted

Signed-off-by: Dominik Hoffmann <dominik151099@outlook.de>

src/sdk/python/rtdip_sdk/pipelines/data_quality/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/interfaces.py

@@ -16,7 +16,7 @@
 
 from pyspark.sql import DataFrame
 
-from rtdip_sdk.pipelines.interfaces import PipelineComponentBaseInterface
+from src.sdk.python.rtdip_sdk.pipelines.interfaces import PipelineComponentBaseInterface
 
 
 class DataManipulationBaseInterface(PipelineComponentBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/duplicate_detection.py

@@ -15,7 +15,10 @@
 from pyspark.sql import DataFrame as PySparkDataFrame
 
 from ...interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class DuplicateDetection(DataManipulationBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/interval_filtering.py

@@ -18,11 +18,14 @@
 from pyspark.sql import SparkSession
 from pyspark.sql import DataFrame
 
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 from ...interfaces import DataManipulationBaseInterface
 
 
-class IntervalFiltering (DataManipulationBaseInterface):
+class IntervalFiltering(DataManipulationBaseInterface):
     """
     Cleanses a DataFrame by removing rows outside a specified interval window.
     Example:

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/k_sigma_anomaly_detection.py

@@ -15,7 +15,10 @@
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.functions import mean, stddev, abs, col
 from ...interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class KSigmaAnomalyDetection(DataManipulationBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/missing_value_imputation.py

@@ -21,7 +21,10 @@
 from datetime import timedelta
 from typing import List
 from ...interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class MissingValueImputation(DataManipulationBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/normalization/denormalization.py

@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from pyspark.sql import DataFrame as PySparkDataFrame
-from rtdip_sdk.pipelines.data_quality.data_manipulation.interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.interfaces import (
+    DataManipulationBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 from .normalization import (
     NormalizationBaseClass,
 )

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/normalization/normalization.py

@@ -14,8 +14,13 @@
 from abc import abstractmethod
 from pyspark.sql import DataFrame as PySparkDataFrame
 from typing import List
-from rtdip_sdk.pipelines.data_quality.data_manipulation.interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.interfaces import (
+    DataManipulationBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class NormalizationBaseClass(DataManipulationBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/data_quality/prediction/arima.py

@@ -19,7 +19,10 @@
 from statsmodels.tsa.arima.model import ARIMA, ARIMAResults
 
 from ....interfaces import DataManipulationBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class ArimaPrediction(DataManipulationBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/interfaces.py

@@ -16,7 +16,7 @@
 
 from pyspark.sql import DataFrame
 
-from rtdip_sdk.pipelines.interfaces import PipelineComponentBaseInterface
+from src.sdk.python.rtdip_sdk.pipelines.interfaces import PipelineComponentBaseInterface
 
 
 class MonitoringBaseInterface(PipelineComponentBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/check_value_ranges.py

@@ -4,8 +4,13 @@
 from functools import reduce
 from operator import or_
 
-from rtdip_sdk.pipelines.data_quality.monitoring.interfaces import MonitoringBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.interfaces import (
+    MonitoringBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class CheckValueRanges(MonitoringBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/flatline_detection.py

@@ -17,8 +17,13 @@
 from pyspark.sql.functions import col, when, lag, count, sum
 from pyspark.sql.window import Window
 
-from rtdip_sdk.pipelines.data_quality.monitoring.interfaces import MonitoringBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.interfaces import (
+    MonitoringBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 
 
 class FlatlineDetection(MonitoringBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/great_expectations_data_quality.py

@@ -14,8 +14,13 @@
 
 import great_expectations as gx
 from pyspark.sql import DataFrame, SparkSession
-from rtdip_sdk.pipelines.data_quality.monitoring.interfaces import MonitoringBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.interfaces import (
+    MonitoringBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
 from great_expectations.checkpoint import (
     Checkpoint,
 )

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/identify_missing_data_interval.py

@@ -18,9 +18,16 @@
 from pyspark.sql import functions as F
 from pyspark.sql.window import Window
 
-from rtdip_sdk.pipelines.data_quality.monitoring.interfaces import MonitoringBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
-from rtdip_sdk.pipelines.utilities.spark.time_string_parsing import parse_time_string_to_ms
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.interfaces import (
+    MonitoringBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
+from src.sdk.python.rtdip_sdk.pipelines.utilities.spark.time_string_parsing import (
+    parse_time_string_to_ms,
+)
 
 
 class IdentifyMissingDataInterval(MonitoringBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/identify_missing_data_pattern.py

@@ -4,9 +4,16 @@
 from pyspark.sql import DataFrame as PySparkDataFrame
 from pyspark.sql import functions as F
 
-from rtdip_sdk.pipelines.data_quality.monitoring.interfaces import MonitoringBaseInterface
-from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
-from rtdip_sdk.pipelines.utilities.spark.time_string_parsing import parse_time_string_to_ms
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.interfaces import (
+    MonitoringBaseInterface,
+)
+from src.sdk.python.rtdip_sdk.pipelines._pipeline_utils.models import (
+    Libraries,
+    SystemType,
+)
+from src.sdk.python.rtdip_sdk.pipelines.utilities.spark.time_string_parsing import (
+    parse_time_string_to_ms,
+)
 
 
 class IdentifyMissingDataPattern(MonitoringBaseInterface):

tests/sdk/python/rtdip_sdk/pipelines/data_quality/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.

tests/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.

tests/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/test_arima.py

@@ -19,7 +19,9 @@
 from pyspark.sql import SparkSession
 from pyspark.sql.dataframe import DataFrame
 
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.prediction.arima import ArimaPrediction
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.prediction.arima import (
+    ArimaPrediction,
+)
 
 
 @pytest.fixture(scope="session")

tests/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/test_k_sigma_anomaly_detection.py

@@ -1,7 +1,8 @@
 from pyspark.sql import SparkSession
 
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.k_sigma_anomaly_detection import \
-    KSigmaAnomalyDetection
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.k_sigma_anomaly_detection import (
+    KSigmaAnomalyDetection,
+)
 
 # Normal data mean=10 stddev=5 + 3 anomalies
 # fmt: off

tests/sdk/python/rtdip_sdk/pipelines/data_quality/data_manipulation/spark/test_normalization.py

@@ -17,14 +17,18 @@
 from pyspark.sql import SparkSession
 from pyspark.sql.dataframe import DataFrame
 
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.denormalization import \
-    Denormalization
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization import \
-    NormalizationBaseClass
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization_mean import \
-    NormalizationMean
-from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization_minmax import \
-    NormalizationMinMax
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.denormalization import (
+    Denormalization,
+)
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization import (
+    NormalizationBaseClass,
+)
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization_mean import (
+    NormalizationMean,
+)
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.data_manipulation.spark.data_quality.normalization.normalization_minmax import (
+    NormalizationMinMax,
+)
 
 
 @pytest.fixture(scope="session")

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2024 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/test_check_value_ranges.py

@@ -3,7 +3,7 @@
 from io import StringIO
 import logging
 
-from rtdip_sdk.pipelines.data_quality.monitoring.spark.check_value_ranges import (
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.check_value_ranges import (
     CheckValueRanges,
 )
 

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/test_flatline_detection.py

@@ -1,6 +1,6 @@
 import pytest
 from pyspark.sql import SparkSession
-from rtdip_sdk.pipelines.data_quality.monitoring.spark.flatline_detection import (
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.flatline_detection import (
     FlatlineDetection,
 )
 

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/test_great_expectations_data_quality.py

@@ -1,7 +1,7 @@
 from pytest_mock import MockerFixture
 from pyspark.sql import SparkSession
 
-from rtdip_sdk.pipelines.data_quality.monitoring.spark.great_expectations_data_quality import (
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.great_expectations_data_quality import (
     GreatExpectationsDataQuality,
 )
 

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/test_identify_missing_data_interval.py

@@ -1,6 +1,6 @@
 import pytest
 from pyspark.sql import SparkSession
-from rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_interval import (
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_interval import (
     IdentifyMissingDataInterval,
 )
 

tests/sdk/python/rtdip_sdk/pipelines/data_quality/monitoring/spark/test_identify_missing_data_pattern.py

@@ -8,7 +8,7 @@
 from pyspark.sql.types import StructType, StructField, StringType
 
 
-from rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_pattern import (
+from src.sdk.python.rtdip_sdk.pipelines.data_quality.monitoring.spark.identify_missing_data_pattern import (
     IdentifyMissingDataPattern,
 )
 

tests/sdk/python/rtdip_sdk/pipelines/machine_learning/spark/test_linear_regression.py

@@ -170,8 +170,8 @@ def sample_data(spark):
 # Test cases
 def test_cols_to_vector(sample_data):
     df = sample_data
-    # Pass the DataFrame to ColsToVector
-    cols_to_vector = ColsToVector(df=df, input_cols=["Value"], output_col="features")
+    # Pass the DataFrame to ColumnsToVector
+    cols_to_vector = ColumnsToVector(df=df, input_cols=["Value"], output_col="features")
     transformed_df = cols_to_vector.transform()
 
     assert "features" in transformed_df.columns
@@ -180,8 +180,8 @@ def test_cols_to_vector(sample_data):
 
 def test_polynomial_features(sample_data):
     df = sample_data
-    # Convert 'Value' to a vector using ColsToVector
-    cols_to_vector = ColsToVector(df=df, input_cols=["Value"], output_col="features")
+    # Convert 'Value' to a vector using ColumnsToVector
+    cols_to_vector = ColumnsToVector(df=df, input_cols=["Value"], output_col="features")
     vectorized_df = cols_to_vector.transform()
 
     polynomial_features = PolynomialFeatures(
@@ -197,8 +197,8 @@ def test_polynomial_features(sample_data):
 
 def test_linear_regression(sample_data):
     df = sample_data
-    # Use ColsToVector to assemble features into a single vector column
-    cols_to_vector = ColsToVector(df=df, input_cols=["Value"], output_col="features")
+    # Use ColumnsToVector to assemble features into a single vector column
+    cols_to_vector = ColumnsToVector(df=df, input_cols=["Value"], output_col="features")
     df = cols_to_vector.transform()
     linear_regression = LinearRegression(
         df, features_col="features", label_col="Value", prediction_col="prediction"