From 71df2724b12d724a0fbae12a733a9fe70c1080d4 Mon Sep 17 00:00:00 2001 From: Leonard <92177433+LeonardHd@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:47:04 +0100 Subject: [PATCH] fix: evaluate expression from root (#67) * fix: evaluate expressions from root to support branches * chore: vscode-pytest only use tests folder --- .vscode/settings.json | 3 +- .../functions/__init__.py | 2 + .../functions/_expression_transformer.py | 291 -------- .../functions/evaluator/__init__.py | 5 + .../functions/evaluator/exceptions.py | 21 + .../{ => evaluator}/expression_evaluator.py | 66 +- .../evaluator/expression_rule_transformer.py | 73 ++ .../expression_terminal_transformer.py | 80 +++ .../functions/evaluator/rules/__init__.py | 33 + ...ity_reference_expression_rule_evaluator.py | 31 + .../rules/branch_expression_rule_evaluator.py | 47 ++ ...set_reference_expression_rule_evaluator.py | 35 + .../evaluation_expression_rule_evaluator.py | 41 ++ ...ion_parameter_expression_rule_evaluator.py | 29 + .../rules/expression_rule_evaluator.py | 54 ++ ...function_call_expression_rule_evaluator.py | 77 ++ ...tem_reference_expression_rule_evaluator.py | 26 + ...ice_reference_expression_rule_evaluator.py | 32 + ...al_evaluation_expression_rule_evaluator.py | 29 + ...interpolation_expression_rule_evaluator.py | 32 + .../logical_bool_expression_rule_evaluator.py | 68 ++ ...ine_reference_expression_rule_evaluator.py | 39 ++ ...ble_reference_expression_rule_evaluator.py | 35 + ...ble_reference_expression_rule_evaluator.py | 30 + .../functions_logical_implementation.py | 18 - .../functions/functions_repository.py | 3 - .../models/data_factory_element.py | 2 +- .../functions/test_expression_evaluator.py | 661 +++++------------- .../test_functions_logical_implementation.py | 49 -- 29 files changed, 1046 insertions(+), 866 deletions(-) delete mode 100644 src/data_factory_testing_framework/functions/_expression_transformer.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/__init__.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/exceptions.py rename src/data_factory_testing_framework/functions/{ => evaluator}/expression_evaluator.py (70%) create mode 100644 src/data_factory_testing_framework/functions/evaluator/expression_rule_transformer.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/expression_terminal_transformer.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/__init__.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/activity_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/branch_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/dataset_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/evaluation_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/expression_parameter_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/function_call_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/item_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/linked_service_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/literal_evaluation_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/literal_interpolation_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/logical_bool_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/pipeline_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/system_variable_reference_expression_rule_evaluator.py create mode 100644 src/data_factory_testing_framework/functions/evaluator/rules/variable_reference_expression_rule_evaluator.py diff --git a/.vscode/settings.json b/.vscode/settings.json index af9ff4fe..c79e8b8e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,8 @@ // for contributors. { "python.testing.pytestArgs": [ - "." + "tests", + "examples", "-vv" ], "python.testing.cwd": "${workspaceFolder}", diff --git a/src/data_factory_testing_framework/functions/__init__.py b/src/data_factory_testing_framework/functions/__init__.py index c9893be2..b15f3ffd 100644 --- a/src/data_factory_testing_framework/functions/__init__.py +++ b/src/data_factory_testing_framework/functions/__init__.py @@ -1,5 +1,7 @@ +from data_factory_testing_framework.functions.evaluator import ExpressionEvaluator from data_factory_testing_framework.functions.functions_repository import FunctionsRepository __all__ = [ "FunctionsRepository", + "ExpressionEvaluator", ] diff --git a/src/data_factory_testing_framework/functions/_expression_transformer.py b/src/data_factory_testing_framework/functions/_expression_transformer.py deleted file mode 100644 index 38bd9dc2..00000000 --- a/src/data_factory_testing_framework/functions/_expression_transformer.py +++ /dev/null @@ -1,291 +0,0 @@ -import inspect -from typing import Any, Callable, Optional - -from lark import Discard, Token, Transformer, Tree -from lxml.etree import _Element - -from data_factory_testing_framework.exceptions.expression_evaluation_error import ( - ExpressionEvaluationError, -) -from data_factory_testing_framework.exceptions.state_iteration_item_not_set_error import ( - StateIterationItemNotSetError, -) -from data_factory_testing_framework.functions.functions_repository import FunctionsRepository -from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState -from data_factory_testing_framework.state.run_parameter_type import RunParameterType - - -class ExpressionTransformer(Transformer): - def __init__(self, state: PipelineRunState) -> None: - """Transformer for the Expression Language.""" - self.state: PipelineRunState = state - super().__init__() - - def LITERAL_LETTER(self, token: Token) -> str: # noqa: N802 - return str(token.value) - - def LITERAL_INT(self, token: Token) -> int: # noqa: N802 - return int(token.value) - - def LITERAL_FLOAT(self, token: Token) -> float: # noqa: N802 - return float(token.value) - - def LITERAL_SINGLE_QUOTED_STRING(self, token: Token) -> str: # noqa: N802 - return str(token.value) - - def LITERAL_BOOLEAN(self, token: Token) -> bool: # noqa: N802 - return bool(token.value) - - def LITERAL_NULL(self, token: Token) -> Optional[None]: # noqa: N802 - return None - - def literal_evaluation(self, value: list[Token, str, int, float, bool]) -> [str, int, float, bool, None]: - if len(value) != 1: - raise ExpressionEvaluationError("Literal evaluation should have only one value") - if type(value[0]) not in [str, int, float, bool, None]: - raise ExpressionEvaluationError("Literal evaluation only supports string, int, float, bool and None") - return value[0] - - def literal_interpolation(self, value: list[Token, str, int, float, bool]) -> str: - result = "" - for item in value: - if type(item) not in [str, int, float, bool, None]: - raise ExpressionEvaluationError("Literal interpolation only supports string, int, float, bool and None") - - result += str(item) - - return result - - def EXPRESSION_NULL(self, token: Token) -> Optional[None]: # noqa: N802 - return None - - def EXPRESSION_STRING(self, token: Token) -> str: # noqa: N802 - string = str(token.value) - string = string.replace("''", "'") # replace escaped single quotes - string = string[1:-1] - - return string - - def EXPRESSION_INTEGER(self, token: Token) -> int: # noqa: N802 - return int(token.value) - - def EXPRESSION_FLOAT(self, token: Token) -> float: # noqa: N802 - return float(token.value) - - def EXPRESSION_BOOLEAN(self, token: Token) -> bool: # noqa: N802 - return bool(token.value) - - def EXPRESSION_WS(self, token: Token) -> Discard: # noqa: N802 - # Discard whitespaces in expressions - return Discard - - def EXPRESSION_ARRAY_INDEX(self, token: Token) -> int: # noqa: N802 - token.value = int(token.value[1:-1]) - return token - - def expression_pipeline_reference(self, values: list[Token, str, int, float, bool]) -> [str, int, float, bool]: - if len(values) != 2: - raise ExpressionEvaluationError("Pipeline reference should have two values") - - if not (isinstance(values[0], Token) and values[0].type == "EXPRESSION_PIPELINE_PROPERTY"): - raise ExpressionEvaluationError('Pipeline reference requires Token "EXPRESSION_PIPELINE_PROPERTY"') - - if not (isinstance(values[1], Token) and values[1].type == "EXPRESSION_PARAMETER_NAME"): - raise ExpressionEvaluationError('Pipeline reference requires Token "EXPRESSION_PARAMETER_NAME"') - - parameter_name = values[1] - parameter_type = self._parse_run_parameter_type(values[0]) - return self.state.get_parameter_by_type_and_name( - parameter_type, - parameter_name, - ) - - def expression_variable_reference(self, values: list[Token, str, int, float, bool]) -> [str, int, float, bool]: - if len(values) != 1: - raise ExpressionEvaluationError("Variable reference should have one value") - - if not (isinstance(values[0], Token) and values[0].type == "EXPRESSION_VARIABLE_NAME"): - raise ExpressionEvaluationError('Variable reference requires Token "EXPRESSION_VARIABLE_NAME"') - - variable_name = values[0].value - variable_name = variable_name[1:-1] # remove quotes - - variable = self.state.get_variable_by_name(variable_name) - - return variable.value - - def expression_dataset_reference(self, values: list[Token, str, int, float, bool]) -> [str, int, float, bool]: - if len(values) != 1: - raise ExpressionEvaluationError("Dataset reference should have one value") - - if not (isinstance(values[0], Token) and values[0].type == "EXPRESSION_PARAMETER_NAME"): - raise ExpressionEvaluationError('Dataset reference requires Token "EXPRESSION_PARAMETER_NAME"') - - parameter_name = values[0].value - - return self.state.get_parameter_by_type_and_name( - RunParameterType.Dataset, - parameter_name, - ) - - def expression_linked_service_reference( - self, values: list[Token, str, int, float, bool] - ) -> [str, int, float, bool]: - if len(values) != 1: - raise ExpressionEvaluationError("Linked service reference should have one value") - - if not (isinstance(values[0], Token) and values[0].type == "EXPRESSION_PARAMETER_NAME"): - raise ExpressionEvaluationError('Linked service reference requires Token "EXPRESSION_PARAMETER_NAME"') - - parameter_name = values[0].value - - return self.state.get_parameter_by_type_and_name( - RunParameterType.LinkedService, - parameter_name, - ) - - def expression_activity_reference( - self, values: list[Tree, Token, str, int, float, bool] - ) -> [str, int, float, bool]: - if len(values) != 2: - raise ExpressionEvaluationError("Activity reference should have two values") - - expression_activity_name = values[0] - if not isinstance(expression_activity_name, Token): - raise ExpressionEvaluationError("Activity reference requires Token") - - if not expression_activity_name.type == "EXPRESSION_ACTIVITY_NAME": - raise ExpressionEvaluationError('Activity reference requires Token "EXPRESSION_ACTIVITY_NAME"') - - activity_name = expression_activity_name.value - activity_name = activity_name[1:-1] # remove quotes - - activity = self.state.get_activity_result_by_name(activity_name) - - return self._evaluate_expression_object_accessors(activity, [values[1]]) - - def expression_item_reference(self, values: list[Tree, Token, str, int, float, bool]) -> [str, int, float, bool]: - if len(values) != 0: - raise ExpressionEvaluationError("Item reference should not have any values") - - item = self.state.iteration_item - if item is None: - raise StateIterationItemNotSetError() - - return item - - def expression_system_variable_reference( - self, values: list[Token, str, int, float, bool] - ) -> [str, int, float, bool]: - if len(values) != 1: - raise ExpressionEvaluationError("System variable reference should have one value") - - if not (isinstance(values[0], Token) and values[0].type == "EXPRESSION_SYSTEM_VARIABLE_NAME"): - raise ExpressionEvaluationError( - 'System variable reference requires Token "EXPRESSION_SYSTEM_VARIABLE_NAME"' - ) - - system_variable_name: Token = values[0] - system_variable = self.state.get_parameter_by_type_and_name( - RunParameterType.System, - system_variable_name, - ) - - return system_variable - - def expression_function_parameters(self, values: list[Token, str, int, float, bool]) -> list: - if not all(type(value) in [str, int, float, bool, list, _Element] or value is None for value in values): - raise ExpressionEvaluationError("Function parameters should be string, int, float, bool, list or _Element") - return values - - def expression_parameter(self, values: list[Token, str, int, float, bool, list]) -> str: - if len(values) != 1: - raise ExpressionEvaluationError("Function parameter must have only one value") - - parameter = values[0] - - if type(parameter) not in [str, int, float, bool, list, _Element, None] and parameter is not None: - raise ExpressionEvaluationError("Function parameters should be string, int, float, bool, list or _Element") - return parameter - - def expression_evaluation(self, values: list[Token, str, int, float, bool, list]) -> [str, int, float, bool]: - eval_value = values[0] - - remaining_fields = values[1:] - return self._evaluate_expression_object_accessors(eval_value, remaining_fields) - - def expression_interpolation_evaluation( - self, values: list[Token, str, int, float, bool, list] - ) -> [str, int, float, bool]: - if len(values) != 1: - raise ExpressionEvaluationError("Interpolation evaluation should have one value") - - return values[0] - - def expression_array_indices(self, values: list[Token, str, int, float, bool]) -> Optional[list[Token]]: - if values[0] is None: - return Discard # if there are no array indices, discard the value - if not all(isinstance(value, Token) and value.type == "EXPRESSION_ARRAY_INDEX" for value in values): - raise ExpressionEvaluationError('Array indices should be of type "EXPRESSION_ARRAY_INDEX"') - return values - - def expression_function_call(self, values: list[Token, str, int, float, bool]) -> [str, int, float, bool]: - fn = values[0] - fn_parameters = values[1] if len(values) == 2 and values[1] is not None else [] - function: Callable = FunctionsRepository.functions.get(fn.value) - - pos_or_keyword_parameters = [] - - function_signature = inspect.signature(function) - pos_or_keyword_parameters = [ - param - for param in function_signature.parameters.values() - if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - ] - - pos_or_keyword_values = fn_parameters[: len(pos_or_keyword_parameters)] - var_positional_values = fn_parameters[len(pos_or_keyword_parameters) :] # should be 0 or 1 - # TODO: implement automatic conversion of parameters based on type hints - result = function(*pos_or_keyword_values, *var_positional_values) - - return result - - @staticmethod - def _evaluate_expression_object_accessors(current_item: Any, expression_object_accessors: list[Tree]) -> Any: # noqa: ANN401 - if not all( - isinstance(expression_object_accessor, Tree) - and isinstance(expression_object_accessor.data, Token) - and expression_object_accessor.data.value == "expression_object_accessor" - for expression_object_accessor in expression_object_accessors - ): - raise ExpressionEvaluationError('Fields should be of type "expression_object_accessor"') - - for expression_object_accessor in expression_object_accessors: - for field_token in expression_object_accessor.children: - if not isinstance(field_token, Token): - raise ExpressionEvaluationError('Fields should be of type "Token"') - if field_token.type == "EXPRESSION_PARAMETER_NAME": - if field_token.value not in current_item: - raise ValueError(field_token.value) - - current_item = current_item[field_token.value] - elif field_token.type == "EXPRESSION_ARRAY_INDEX": - if not isinstance(current_item, list): - raise ExpressionEvaluationError("Array index can only be used on lists") - - current_item = current_item[field_token.value] - else: - raise ExpressionEvaluationError( - 'Fields should be of type "EXPRESSION_PARAMETER_NAME" or "EXPRESSION_ARRAY_INDEX"' - ) - - return current_item - - @staticmethod - def _parse_run_parameter_type(run_parameter_type: str) -> RunParameterType: - if run_parameter_type == "parameters": - return RunParameterType.Pipeline - elif run_parameter_type == "globalParameters": - return RunParameterType.Global - else: - raise ValueError(f"Unsupported run parameter type: {run_parameter_type}") diff --git a/src/data_factory_testing_framework/functions/evaluator/__init__.py b/src/data_factory_testing_framework/functions/evaluator/__init__.py new file mode 100644 index 00000000..c0597009 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/__init__.py @@ -0,0 +1,5 @@ +from .expression_evaluator import ExpressionEvaluator + +__all__ = [ + "ExpressionEvaluator", +] diff --git a/src/data_factory_testing_framework/functions/evaluator/exceptions.py b/src/data_factory_testing_framework/functions/evaluator/exceptions.py new file mode 100644 index 00000000..c8daae40 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/exceptions.py @@ -0,0 +1,21 @@ +from typing import Union + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError + + +class ExpressionEvaluationInvalidNumberOfChildrenError(ExpressionEvaluationError): + """Expression evaluation invalid number of children error.""" + + def __init__(self, required: int, actual: int) -> None: + """Initialize expression evaluation invalid number of children error.""" + super().__init__(f"Invalid number of children. Required: {required}, Actual: {actual}") + + +class ExpressionEvaluationInvalidChildTypeError(ExpressionEvaluationError): + """Expression evaluation invalid child type error.""" + + def __init__(self, child_index: int, expected_types: Union[tuple[type], type], actual_type: type) -> None: + """Initialize expression evaluation invalid child type error.""" + super().__init__( + f"Invalid child type at index {child_index}. Expected: {expected_types}, Actual: {actual_type}" + ) diff --git a/src/data_factory_testing_framework/functions/expression_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/expression_evaluator.py similarity index 70% rename from src/data_factory_testing_framework/functions/expression_evaluator.py rename to src/data_factory_testing_framework/functions/evaluator/expression_evaluator.py index 525b460e..a4fc8ae7 100644 --- a/src/data_factory_testing_framework/functions/expression_evaluator.py +++ b/src/data_factory_testing_framework/functions/evaluator/expression_evaluator.py @@ -1,10 +1,17 @@ from typing import Union from lark import Lark, Token, Tree, UnexpectedCharacters -from lark.exceptions import VisitError +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError from data_factory_testing_framework.exceptions.expression_parsing_error import ExpressionParsingError -from data_factory_testing_framework.functions._expression_transformer import ExpressionTransformer +from data_factory_testing_framework.functions.evaluator.expression_rule_transformer import ( + ExpressionRuleTransformer, +) +from data_factory_testing_framework.functions.evaluator.expression_terminal_transformer import ( + ExpressionTerminalTransformer, +) +from data_factory_testing_framework.functions.evaluator.rules import ExpressionRuleEvaluator +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult from data_factory_testing_framework.functions.functions_repository import FunctionsRepository from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState @@ -39,7 +46,8 @@ def __init__(self) -> None: expression_grammar = f""" // TODO: add support for array index ?expression_start: expression_evaluation - expression_evaluation: expression_call [expression_object_accessor]* + # TODO: probably object accessor does not apply to all below in expression_evaluation + expression_evaluation: (expression_logical_bool | expression_branch | expression_call) ((("." EXPRESSION_PARAMETER_NAME) | EXPRESSION_ARRAY_INDEX)+)? ?expression_call: expression_function_call | expression_pipeline_reference | expression_variable_reference @@ -48,37 +56,40 @@ def __init__(self) -> None: | expression_linked_service_reference | expression_item_reference | expression_system_variable_reference - expression_object_accessor: ["." EXPRESSION_PARAMETER_NAME] | [EXPRESSION_ARRAY_INDEX] // reference rules: expression_pipeline_reference: "pipeline" "()" "." EXPRESSION_PIPELINE_PROPERTY "." EXPRESSION_PARAMETER_NAME expression_variable_reference: "variables" "(" EXPRESSION_VARIABLE_NAME ")" - expression_activity_reference: "activity" "(" EXPRESSION_ACTIVITY_NAME ")" expression_object_accessor + expression_activity_reference: "activity" "(" EXPRESSION_ACTIVITY_NAME ")" expression_dataset_reference: "dataset" "()" "." EXPRESSION_PARAMETER_NAME expression_linked_service_reference: "linkedService" "()" "." EXPRESSION_PARAMETER_NAME expression_item_reference: "item" "()" expression_system_variable_reference: "pipeline" "()" "." EXPRESSION_SYSTEM_VARIABLE_NAME + // branch rules + expression_logical_bool: EXPRESSION_LOGICAL_BOOL "(" expression_parameter "," expression_parameter ")" + expression_branch: "if" "(" expression_parameter "," expression_parameter "," expression_parameter ")" + // function call rules - expression_function_call: EXPRESSION_FUNCTION_NAME "(" [expression_function_parameters] ")" - expression_function_parameters: expression_parameter ("," expression_parameter )* - expression_parameter: EXPRESSION_WS* (EXPRESSION_NULL | EXPRESSION_INTEGER | EXPRESSION_FLOAT | EXPRESSION_BOOLEAN | EXPRESSION_STRING | expression_start) EXPRESSION_WS* + expression_function_call: EXPRESSION_FUNCTION_NAME "(" [expression_parameter ("," expression_parameter )*] ")" + ?expression_parameter: EXPRESSION_WS* (EXPRESSION_NULL | EXPRESSION_INTEGER | EXPRESSION_FLOAT | EXPRESSION_BOOLEAN | EXPRESSION_STRING | expression_start) EXPRESSION_WS* // expression terminals // EXPRESSION_PIPELINE_PROPERTY requires higher priority, because it clashes with pipeline().system_variable.field in the rule: expression_pipeline_reference - EXPRESSION_PIPELINE_PROPERTY.2: "parameters" | "globalParameters" - EXPRESSION_PARAMETER_NAME: /[a-zA-Z0-9_]+/ - EXPRESSION_VARIABLE_NAME: "'" /[^']*/ "'" EXPRESSION_ACTIVITY_NAME: "'" /[^']*/ "'" - EXPRESSION_SYSTEM_VARIABLE_NAME: /[a-zA-Z0-9_]+/ + EXPRESSION_ARRAY_INDEX: ARRAY_INDEX + EXPRESSION_BOOLEAN: BOOLEAN + EXPRESSION_FLOAT: SIGNED_FLOAT EXPRESSION_FUNCTION_NAME: {self._supported_functions()} + EXPRESSION_INTEGER: SIGNED_INT + EXPRESSION_LOGICAL_BOOL: "or" | "and" EXPRESSION_NULL: NULL + EXPRESSION_PARAMETER_NAME: /[a-zA-Z0-9_]+/ + EXPRESSION_PIPELINE_PROPERTY.2: "parameters" | "globalParameters" EXPRESSION_STRING: SINGLE_QUOTED_STRING - EXPRESSION_INTEGER: SIGNED_INT - EXPRESSION_FLOAT: SIGNED_FLOAT - EXPRESSION_BOOLEAN: BOOLEAN + EXPRESSION_SYSTEM_VARIABLE_NAME: /[a-zA-Z0-9_]+/ + EXPRESSION_VARIABLE_NAME: "'" /[^']*/ "'" EXPRESSION_WS: WS - EXPRESSION_ARRAY_INDEX: ARRAY_INDEX """ # noqa: E501 base_grammar = """ @@ -105,13 +116,14 @@ def _supported_functions(self) -> str: functions = [f'"{f}"' for f in functions] return " | ".join(functions) - def parse(self, expression: str) -> Tree[Token]: + def _parse(self, expression: str) -> Tree[Token]: tree = self.lark_parser.parse(expression) return tree def evaluate(self, expression: str, state: PipelineRunState) -> Union[str, int, float, bool]: try: - tree = self.parse(expression) + parse_tree = self._parse(expression) + except UnexpectedCharacters as uc: msg = f""" Expression could not be parsed. @@ -122,10 +134,14 @@ def evaluate(self, expression: str, state: PipelineRunState) -> Union[str, int, char: {uc.char} """ raise ExpressionParsingError(msg) from uc - - transformer = ExpressionTransformer(state) - try: - result: Tree = transformer.transform(tree) - except VisitError as ve: - raise ve.orig_exc from ve - return result + # we start with a raw parse tree for the lark grammer + # and then semantically analysis it (in our case transforming values step by step) + ast = parse_tree + ast = ExpressionTerminalTransformer().transform(ast) + ast = ExpressionRuleTransformer(state).transform(ast) + if not isinstance(ast, ExpressionRuleEvaluator): + raise ExpressionEvaluationError() + result = ast.evaluate() + if not isinstance(result, EvaluationResult): + raise ExpressionEvaluationError() + return result.value diff --git a/src/data_factory_testing_framework/functions/evaluator/expression_rule_transformer.py b/src/data_factory_testing_framework/functions/evaluator/expression_rule_transformer.py new file mode 100644 index 00000000..48ee4ab7 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/expression_rule_transformer.py @@ -0,0 +1,73 @@ +from lark import Transformer, Tree, v_args + +from data_factory_testing_framework.functions.evaluator.rules import ( + ActivityReferenceExpressionRuleEvaluator, + BranchExpressionRuleEvaluator, + DatasetReferenceExpressionRuleEvaluator, + EvaluationExpressionRuleEvaluator, + ExpressionParameterExpressionRuleEvaluator, + ExpressionRuleEvaluator, + FunctionCallExpressionRuleEvaluator, + ItemReferenceExpressionRuleEvaluator, + LinkedServiceReferenceExpressionRuleEvaluator, + LiteralEvaluationExpressionRuleEvaluator, + LiteralInterpolationExpressionRuleEvaluator, + LogicalBoolExpressionEvaluator, + PipelineReferenceExpressionRuleEvaluator, + SystemVariableReferenceExpressionRuleEvaluator, + VariableReferenceExpressionRuleEvaluator, +) +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState + + +@v_args(tree=True) +class ExpressionRuleTransformer(Transformer[ExpressionRuleEvaluator]): + def __init__(self, state: PipelineRunState) -> None: # noqa: D107 + visit_tokens = False + self.state: PipelineRunState = state + super().__init__(visit_tokens) + + def expression_pipeline_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return PipelineReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_logical_bool(self, tree: Tree) -> ExpressionRuleEvaluator: + return LogicalBoolExpressionEvaluator(tree) + + def expression_branch(self, tree: Tree) -> ExpressionRuleEvaluator: + return BranchExpressionRuleEvaluator(tree) + + def expression_evaluation(self, tree: Tree) -> ExpressionRuleEvaluator: + return EvaluationExpressionRuleEvaluator(tree) + + def expression_function_call(self, tree: Tree) -> ExpressionRuleEvaluator: + return FunctionCallExpressionRuleEvaluator(tree) + + def expression_item_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return ItemReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_system_variable_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return SystemVariableReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_activity_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return ActivityReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_linked_service_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return LinkedServiceReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_dataset_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return DatasetReferenceExpressionRuleEvaluator(tree, self.state) + + def expression_variable_reference(self, tree: Tree) -> ExpressionRuleEvaluator: + return VariableReferenceExpressionRuleEvaluator(tree, self.state) + + def literal_interpolation(self, tree: Tree) -> ExpressionRuleEvaluator: + return LiteralInterpolationExpressionRuleEvaluator(tree) + + def literal_evaluation(self, tree: Tree) -> ExpressionRuleEvaluator: + return LiteralEvaluationExpressionRuleEvaluator(tree) + + def expression_parameter(self, tree: Tree) -> ExpressionRuleEvaluator: + return ExpressionParameterExpressionRuleEvaluator(tree) + + def __default__(self, data, children, meta): # noqa: ANN204, D105, ANN001 + raise ValueError(f"Unknown expression rule with data: {data}, children: {children}, meta: {meta}") diff --git a/src/data_factory_testing_framework/functions/evaluator/expression_terminal_transformer.py b/src/data_factory_testing_framework/functions/evaluator/expression_terminal_transformer.py new file mode 100644 index 00000000..cfb4e1e7 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/expression_terminal_transformer.py @@ -0,0 +1,80 @@ +# ruff: noqa: N802 + + +from lark import Discard, Token, Transformer + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.run_parameter_type import RunParameterType + + +class ExpressionTerminalTransformer(Transformer): + def EXPRESSION_ACTIVITY_NAME(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value[1:-1]) + + def EXPRESSION_ARRAY_INDEX(self, token: Token) -> EvaluationResult: + return EvaluationResult(int(token.value[1:-1])) + + def EXPRESSION_BOOLEAN(self, token: Token) -> EvaluationResult: + return EvaluationResult(bool(token.value)) + + def EXPRESSION_FLOAT(self, token: Token) -> EvaluationResult: + return EvaluationResult(float(token.value)) + + def EXPRESSION_FUNCTION_NAME(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value) + + def EXPRESSION_INTEGER(self, token: Token) -> EvaluationResult: + return EvaluationResult(int(token.value)) + + def EXPRESSION_LOGICAL_BOOL(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value) + + def EXPRESSION_NULL(self, token: Token) -> EvaluationResult: + return EvaluationResult(None) + + def EXPRESSION_PARAMETER_NAME(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value) + + def EXPRESSION_PIPELINE_PROPERTY(self, token: Token) -> EvaluationResult: + if token.value == "parameters": + return EvaluationResult(RunParameterType.Pipeline) + elif token.value == "globalParameters": + return EvaluationResult(RunParameterType.Global) + else: + raise ExpressionEvaluationError(f"Unsupported run parameter type: {token.value}") + + def EXPRESSION_STRING(self, token: Token) -> EvaluationResult: + string = str(token.value) + string = string.replace("''", "'") # replace escaped single quotes + string = string[1:-1] + + return EvaluationResult(string) + + def EXPRESSION_SYSTEM_VARIABLE_NAME(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value) + + def EXPRESSION_VARIABLE_NAME(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value[1:-1]) + + def EXPRESSION_WS(self, token: Token) -> Discard: + # Discard whitespaces in expressions + return Discard + + def LITERAL_LETTER(self, token: Token) -> EvaluationResult: + return EvaluationResult(token.value) + + def LITERAL_INT(self, token: Token) -> EvaluationResult: + return EvaluationResult(int(token.value)) + + def LITERAL_FLOAT(self, token: Token) -> EvaluationResult: + return EvaluationResult(float(token.value)) + + def LITERAL_SINGLE_QUOTED_STRING(self, token: Token) -> EvaluationResult: + return EvaluationResult(str(token.value)) + + def LITERAL_BOOLEAN(self, token: Token) -> EvaluationResult: + return EvaluationResult(bool(token.value)) + + def LITERAL_NULL(self, token: Token) -> EvaluationResult: + return EvaluationResult(None) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/__init__.py b/src/data_factory_testing_framework/functions/evaluator/rules/__init__.py new file mode 100644 index 00000000..11847c8e --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/__init__.py @@ -0,0 +1,33 @@ +from .activity_reference_expression_rule_evaluator import ActivityReferenceExpressionRuleEvaluator +from .branch_expression_rule_evaluator import BranchExpressionRuleEvaluator +from .dataset_reference_expression_rule_evaluator import DatasetReferenceExpressionRuleEvaluator +from .evaluation_expression_rule_evaluator import EvaluationExpressionRuleEvaluator +from .expression_parameter_expression_rule_evaluator import ExpressionParameterExpressionRuleEvaluator +from .expression_rule_evaluator import ExpressionRuleEvaluator +from .function_call_expression_rule_evaluator import FunctionCallExpressionRuleEvaluator +from .item_reference_expression_rule_evaluator import ItemReferenceExpressionRuleEvaluator +from .linked_service_reference_expression_rule_evaluator import LinkedServiceReferenceExpressionRuleEvaluator +from .literal_evaluation_expression_rule_evaluator import LiteralEvaluationExpressionRuleEvaluator +from .literal_interpolation_expression_rule_evaluator import LiteralInterpolationExpressionRuleEvaluator +from .logical_bool_expression_rule_evaluator import LogicalBoolExpressionEvaluator +from .pipeline_reference_expression_rule_evaluator import PipelineReferenceExpressionRuleEvaluator +from .system_variable_reference_expression_rule_evaluator import SystemVariableReferenceExpressionRuleEvaluator +from .variable_reference_expression_rule_evaluator import VariableReferenceExpressionRuleEvaluator + +__all__ = [ + "ActivityReferenceExpressionRuleEvaluator", + "BranchExpressionRuleEvaluator", + "DatasetReferenceExpressionRuleEvaluator", + "EvaluationExpressionRuleEvaluator", + "ExpressionParameterExpressionRuleEvaluator", + "ExpressionRuleEvaluator", + "FunctionCallExpressionRuleEvaluator", + "ItemReferenceExpressionRuleEvaluator", + "LinkedServiceReferenceExpressionRuleEvaluator", + "LiteralEvaluationExpressionRuleEvaluator", + "LiteralInterpolationExpressionRuleEvaluator", + "LogicalBoolExpressionEvaluator", + "PipelineReferenceExpressionRuleEvaluator", + "SystemVariableReferenceExpressionRuleEvaluator", + "VariableReferenceExpressionRuleEvaluator", +] diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/activity_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/activity_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..882bb4e7 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/activity_reference_expression_rule_evaluator.py @@ -0,0 +1,31 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class ActivityReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize expression rule evaluator.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + self.activity_name = self.children[0].value + + def evaluate(self) -> EvaluationResult: + activity_result = self.state.get_activity_result_by_name(self.activity_name) + return EvaluationResult(activity_result) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/branch_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/branch_expression_rule_evaluator.py new file mode 100644 index 00000000..f1d0f9d2 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/branch_expression_rule_evaluator.py @@ -0,0 +1,47 @@ +from typing import Union + +from lark import Tree + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class BranchExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) != 3: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=3, actual=len(self.children)) + + for i, child in enumerate(self.children): + self._check_child_type(child, i) + + self.condition = self.children[0] + self.true_expression_branch = self.children[1] + self.false_expression_branch = self.children[2] + + def _check_child_type(self, child: Union[EvaluationResult, ExpressionRuleEvaluator], child_index: int) -> None: + if not isinstance(child, (EvaluationResult, ExpressionRuleEvaluator)): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=child_index, + expected_types=(EvaluationResult, ExpressionRuleEvaluator), + actual_type=type(child), + ) + + def evaluate(self) -> EvaluationResult: + condition_result = self.evaluate_child(self.condition) + + if not isinstance(condition_result.value, bool): + raise ExpressionEvaluationError("Expression result must be a boolean value.") + + if condition_result.value: + return self.evaluate_child(self.true_expression_branch) + else: + return self.evaluate_child(self.false_expression_branch) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/dataset_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/dataset_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..053c3dd7 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/dataset_reference_expression_rule_evaluator.py @@ -0,0 +1,35 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState +from data_factory_testing_framework.state.run_parameter_type import RunParameterType + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class DatasetReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize expression rule evaluator.""" + self.state: PipelineRunState = state + super().__init__(tree) + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + self.parameter_name = self.children[0].value + + def evaluate(self) -> EvaluationResult: + result = self.state.get_parameter_by_type_and_name( + RunParameterType.Dataset, + self.parameter_name, + ) + return EvaluationResult(result) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/evaluation_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/evaluation_expression_rule_evaluator.py new file mode 100644 index 00000000..a6b0b21d --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/evaluation_expression_rule_evaluator.py @@ -0,0 +1,41 @@ +from lark import Tree + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.exceptions import ExpressionEvaluationInvalidChildTypeError +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class EvaluationExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) < 1: + raise ExpressionEvaluationError( + f"Invalid number of children. Minimum required: 1, Actual: {len(self.children)}" + ) + + if not isinstance(self.children[0], ExpressionRuleEvaluator): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=ExpressionRuleEvaluator, actual_type=type(self.children[0]) + ) + + for i, child in enumerate(self.children[1:]): + if not isinstance(child, EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=i + 1, expected_types=EvaluationResult, actual_type=type(child) + ) + + self.expression = self.children[0] + self.accessors = self.children[1:] + + def evaluate(self) -> EvaluationResult: + expression_value = self.evaluate_child(self.expression) + + current_value = expression_value.value + for accessor in self.accessors: + current_value = current_value[accessor.value] + + return EvaluationResult(current_value) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/expression_parameter_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/expression_parameter_expression_rule_evaluator.py new file mode 100644 index 00000000..cea0c079 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/expression_parameter_expression_rule_evaluator.py @@ -0,0 +1,29 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class ExpressionParameterExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], (EvaluationResult, ExpressionRuleEvaluator)): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, + expected_types=(EvaluationResult, ExpressionRuleEvaluator), + actual_type=type(self.children[0]), + ) + self.parameter_name_expression = self.children[0] + + def evaluate(self) -> EvaluationResult: + return self.evaluate_child(self.parameter_name_expression) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/expression_rule_evaluator.py new file mode 100644 index 00000000..747ca0e2 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/expression_rule_evaluator.py @@ -0,0 +1,54 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Union + +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ExpressionEvaluationInvalidChildTypeError + + +@dataclass +class EvaluationResult: + """Evaluated expression data class.""" + + value: Any + + +class ExpressionRuleEvaluator(ABC): + """Abstract class for expression rule evaluators. + + Implementations of this class should evaluate the expression rule and return an evaluated expression. + Decorates the lark Tree class to provide a tree structure for the expression rule. + The base class implementation checks that the children are of the correct type. + """ + + def __init__(self, tree: Tree["ExpressionRuleEvaluator"]) -> None: + """Initializes the expression rule.""" + self._tree = tree + + # check that children are of the correct type + if not all(isinstance(child, (EvaluationResult, ExpressionRuleEvaluator)) for child in self.children): + raise ExpressionEvaluationInvalidChildTypeError( + expected_types=(EvaluationResult, ExpressionRuleEvaluator), + actual_types=[type(child) for child in self.children], + ) + + @abstractmethod + def evaluate(self) -> EvaluationResult: + """Evaluates the expression rule.""" + pass + + @property + def children(self) -> list[Union["ExpressionRuleEvaluator", EvaluationResult]]: + """Returns the children of the expression rule.""" + return self._tree.children + + def evaluate_child(self, child: Union[EvaluationResult, "ExpressionRuleEvaluator"]) -> EvaluationResult: + """Evaluates a child of the expression rule. + + If the child is an expression rule evaluator, it is evaluated so that the result is always an EvaluationResult. + """ + if isinstance(child, ExpressionRuleEvaluator): + return child.evaluate() + else: + return child diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/function_call_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/function_call_expression_rule_evaluator.py new file mode 100644 index 00000000..7e71e67a --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/function_call_expression_rule_evaluator.py @@ -0,0 +1,77 @@ +import inspect +from typing import Callable, Union + +from lark import Tree + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.exceptions import ExpressionEvaluationInvalidChildTypeError +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import ( + EvaluationResult, + ExpressionRuleEvaluator, +) +from data_factory_testing_framework.functions.functions_repository import FunctionsRepository + + +class FunctionCallExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) < 1: + raise ExpressionEvaluationError( + f"Invalid number of children. Minimum required: 1, Actual: {len(self.children)}" + ) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + for i, child in enumerate(self.children[1:]): + if isinstance(child, (EvaluationResult, ExpressionRuleEvaluator)): + continue + + raise ExpressionEvaluationInvalidChildTypeError( + child_index=i, expected_types=(EvaluationResult, ExpressionRuleEvaluator), actual_type=type(child) + ) + + self.function_name = self.children[0].value + self.parameters = self.children[1:] + + def evaluate(self) -> EvaluationResult: + evaluated_parameters = self._evaluated_parameters(self.parameters) + function: Callable = FunctionsRepository.functions.get(self.function_name) + + pos_or_kw_params, var_pos_params = self._build_function_call_parameters(function, evaluated_parameters) + result = function(*pos_or_kw_params, *var_pos_params) + + return EvaluationResult(result) + + def _build_function_call_parameters( + self, function: Callable, parameters: list[Union[EvaluationResult, ExpressionRuleEvaluator]] + ) -> tuple[ + Union[ + list[Union[EvaluationResult, ExpressionRuleEvaluator]], + list[Union[EvaluationResult, ExpressionRuleEvaluator]], + ] + ]: + function_signature = inspect.signature(function) + pos_or_keyword_parameters = [ + param + for param in function_signature.parameters.values() + if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + ] + + pos_or_keyword_values = parameters[: len(pos_or_keyword_parameters)] + var_positional_values = parameters[len(pos_or_keyword_parameters) :] + # TODO: implement automatic conversion of parameters based on type hints + return pos_or_keyword_values, var_positional_values + + def _evaluated_parameters( + self, parameters: list[Union[EvaluationResult, ExpressionRuleEvaluator]] + ) -> list[EvaluationResult]: + evaluated_parameters = [] + for p in parameters: + evaluated_expression = self.evaluate_child(p) + evaluated_parameters.append(evaluated_expression.value) + return evaluated_parameters diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/item_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/item_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..bde92590 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/item_reference_expression_rule_evaluator.py @@ -0,0 +1,26 @@ +from lark import Tree + +from data_factory_testing_framework.exceptions.state_iteration_item_not_set_error import StateIterationItemNotSetError +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class ItemReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize expression rule evaluator.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 0: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=0, actual=len(self.children)) + + def evaluate(self) -> EvaluationResult: + item = self.state.iteration_item + if item is None: + raise StateIterationItemNotSetError() + return EvaluationResult(item) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/linked_service_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/linked_service_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..974b9854 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/linked_service_reference_expression_rule_evaluator.py @@ -0,0 +1,32 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState +from data_factory_testing_framework.state.run_parameter_type import RunParameterType + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class LinkedServiceReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize expression rule evaluator.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + self.parameter_name = self.children[0].value + + def evaluate(self) -> EvaluationResult: + parameter = self.state.get_parameter_by_type_and_name(RunParameterType.LinkedService, self.parameter_name) + return EvaluationResult(parameter) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/literal_evaluation_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/literal_evaluation_expression_rule_evaluator.py new file mode 100644 index 00000000..5785ec5d --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/literal_evaluation_expression_rule_evaluator.py @@ -0,0 +1,29 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class LiteralEvaluationExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], (EvaluationResult, ExpressionRuleEvaluator)): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, + expected_types=(EvaluationResult, ExpressionRuleEvaluator), + actual_type=type(self.children[0]), + ) + self.literal = self.children[0].value + + def evaluate(self) -> EvaluationResult: + return EvaluationResult(self.literal) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/literal_interpolation_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/literal_interpolation_expression_rule_evaluator.py new file mode 100644 index 00000000..bcbceecc --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/literal_interpolation_expression_rule_evaluator.py @@ -0,0 +1,32 @@ +from lark import Tree + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.exceptions import ExpressionEvaluationInvalidChildTypeError +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class LiteralInterpolationExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + for index, child in enumerate(self.children): + if not isinstance(child, (ExpressionRuleEvaluator, EvaluationResult)): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=index, + expected_types=(ExpressionRuleEvaluator, EvaluationResult), + actual_type=type(child), + ) + + self.literal_interpolation_items = self.children + + def evaluate(self) -> EvaluationResult: + evaluation_result = "" + for item in self.literal_interpolation_items: + value = self.evaluate_child(item).value + if not isinstance(value, (str, int, float, bool, None)): + raise ExpressionEvaluationError("Literal interpolation only supports string, int, float, bool and None") + evaluation_result += str(value) + return EvaluationResult(evaluation_result) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/logical_bool_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/logical_bool_expression_rule_evaluator.py new file mode 100644 index 00000000..792738b8 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/logical_bool_expression_rule_evaluator.py @@ -0,0 +1,68 @@ +from typing import Union + +from lark import Tree + +from data_factory_testing_framework.exceptions.expression_evaluation_error import ExpressionEvaluationError +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class LogicalBoolExpressionEvaluator(ExpressionRuleEvaluator): + OR = "or" + AND = "and" + + def __init__(self, tree: Tree) -> None: + """Initializes the expression rule evaluator.""" + super().__init__(tree) + + if len(self.children) != 3: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=3, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, + expected_types=(EvaluationResult,), + actual_type=type(self.children[0]), + ) + + for i, child in enumerate(self.children[1:]): + self._check_child_type(child, i) + + if self.children[0].value not in (self.OR, self.AND): + self._raise_invalid_operator(self.children[0].value) + + self.logical_operator = self.children[0].value + self.left_expression = self.children[1] + self.right_expression = self.children[2] + + def _raise_invalid_operator(self, logical_operator: str) -> None: + raise ExpressionEvaluationError(f"Invalid logical operator: {logical_operator}") + + def _check_child_type(self, child: Union[EvaluationResult, ExpressionRuleEvaluator], child_index: int) -> None: + if not isinstance(child, (ExpressionRuleEvaluator, EvaluationResult)): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=child_index, + expected_types=(ExpressionRuleEvaluator, EvaluationResult), + actual_type=type(child), + ) + + def evaluate(self) -> EvaluationResult: + if self.logical_operator == self.OR: + value = self._evaluate_expression(self.left_expression) or self._evaluate_expression(self.right_expression) + elif self.logical_operator == self.AND: + value = self._evaluate_expression(self.left_expression) and self._evaluate_expression(self.right_expression) + else: + self._raise_invalid_operator(self.logical_operator) + return EvaluationResult(value) + + def _evaluate_expression(self, expression: Union[ExpressionRuleEvaluator, EvaluationResult]) -> bool: + result = self.evaluate_child(expression) + if not isinstance(result.value, bool): + raise ExpressionEvaluationError(f"Evaluating expression resulted in non-boolean value: {result.value}") + + return result.value diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/pipeline_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/pipeline_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..0d1d27ff --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/pipeline_reference_expression_rule_evaluator.py @@ -0,0 +1,39 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class PipelineReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState = None) -> None: + """Initialize the expression rule.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 2: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=2, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + if not isinstance(self.children[1], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=1, expected_types=EvaluationResult, actual_type=type(self.children[1]) + ) + self.parameter_type = self.children[0].value + self.parameter_name = self.children[1].value + + def evaluate(self) -> EvaluationResult: + result = self.state.get_parameter_by_type_and_name( + self.parameter_type, + self.parameter_name, + ) + return EvaluationResult(result) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/system_variable_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/system_variable_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..0f29c70c --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/system_variable_reference_expression_rule_evaluator.py @@ -0,0 +1,35 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState +from data_factory_testing_framework.state.run_parameter_type import RunParameterType + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class SystemVariableReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize the expression rule.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + + self.system_variable_name = self.children[0].value + + def evaluate(self) -> EvaluationResult: + system_variable = self.state.get_parameter_by_type_and_name( + RunParameterType.System, + self.system_variable_name, + ) + return EvaluationResult(system_variable) diff --git a/src/data_factory_testing_framework/functions/evaluator/rules/variable_reference_expression_rule_evaluator.py b/src/data_factory_testing_framework/functions/evaluator/rules/variable_reference_expression_rule_evaluator.py new file mode 100644 index 00000000..5d1c7ca4 --- /dev/null +++ b/src/data_factory_testing_framework/functions/evaluator/rules/variable_reference_expression_rule_evaluator.py @@ -0,0 +1,30 @@ +from lark import Tree + +from data_factory_testing_framework.functions.evaluator.exceptions import ( + ExpressionEvaluationInvalidChildTypeError, + ExpressionEvaluationInvalidNumberOfChildrenError, +) +from data_factory_testing_framework.functions.evaluator.rules.expression_rule_evaluator import EvaluationResult +from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState + +from .expression_rule_evaluator import ExpressionRuleEvaluator + + +class VariableReferenceExpressionRuleEvaluator(ExpressionRuleEvaluator): + def __init__(self, tree: Tree, state: PipelineRunState) -> None: + """Initialize expression rule evaluator.""" + super().__init__(tree) + self.state: PipelineRunState = state + + if len(self.children) != 1: + raise ExpressionEvaluationInvalidNumberOfChildrenError(required=1, actual=len(self.children)) + + if not isinstance(self.children[0], EvaluationResult): + raise ExpressionEvaluationInvalidChildTypeError( + child_index=0, expected_types=EvaluationResult, actual_type=type(self.children[0]) + ) + self.variable_name = self.children[0].value + + def evaluate(self) -> EvaluationResult: + variable = self.state.get_variable_by_name(self.variable_name) + return EvaluationResult(variable.value) diff --git a/src/data_factory_testing_framework/functions/functions_logical_implementation.py b/src/data_factory_testing_framework/functions/functions_logical_implementation.py index 4a222e4c..7dddb3f1 100644 --- a/src/data_factory_testing_framework/functions/functions_logical_implementation.py +++ b/src/data_factory_testing_framework/functions/functions_logical_implementation.py @@ -1,14 +1,6 @@ from typing import Any, Union -def and_(expression1: bool, expression2: bool) -> bool: - """Check whether both expressions are true. - - Return true when both expressions are true, or return false when at least one expression is false. - """ - return expression1 and expression2 - - def equals(object1: Any, object2: Any) -> bool: # noqa: ANN401 """Check whether both values, expressions, or objects are equivalent. @@ -33,11 +25,6 @@ def greater_or_equals(value: Any, compare_to: Any) -> bool: # noqa: ANN401 return value >= compare_to -def if_(expression: bool, value_if_true: Any, value_if_false: Any) -> Any: # noqa: ANN401 - """Check whether an expression is true or false. Based on the result, return a specified value.""" - return value_if_true if expression else value_if_false - - def less(value: Union[int, float, str], compare_to: Union[int, float, str]) -> bool: # noqa: ANN401 """Check whether the first value is less than the second value. @@ -57,8 +44,3 @@ def less_or_equals(value: Union[int, float, str], compare_to: Union[int, float, def not_(value: Any) -> bool: # noqa: ANN401 """Check whether an expression is false. Return true when the expression is false, or return false when true.""" return not value - - -def or_(a: Any, b: Any) -> Any: # noqa: ANN401 - """Check whether at least one expression is true. Return true when at least one expression is true, or return false when both are false.""" - return a or b diff --git a/src/data_factory_testing_framework/functions/functions_repository.py b/src/data_factory_testing_framework/functions/functions_repository.py index e9708826..9d9bc999 100644 --- a/src/data_factory_testing_framework/functions/functions_repository.py +++ b/src/data_factory_testing_framework/functions/functions_repository.py @@ -16,7 +16,6 @@ class FunctionsRepository: "addMinutes": date_functions.add_minutes, "addSeconds": date_functions.add_seconds, "addToTime": date_functions.add_to_time, - "and": logical_functions.and_, "array": conversion_functions.array, "base64": conversion_functions.base64, "base64ToBinary": conversion_functions.base64_to_binary, @@ -52,7 +51,6 @@ class FunctionsRepository: "greater": logical_functions.greater, "greaterOrEquals": logical_functions.greater_or_equals, "guid": string_functions.guid, - "if": logical_functions.if_, "indexOf": string_functions.index_of, "int": conversion_functions.int_, "json": conversion_functions.json, @@ -68,7 +66,6 @@ class FunctionsRepository: "mod": math_functions.mod, "mul": math_functions.mul, "not": logical_functions.not_, - "or": logical_functions.or_, "rand": math_functions.rand, "range": math_functions.range_, "replace": string_functions.replace, diff --git a/src/data_factory_testing_framework/models/data_factory_element.py b/src/data_factory_testing_framework/models/data_factory_element.py index 9857e355..9429b5cb 100644 --- a/src/data_factory_testing_framework/models/data_factory_element.py +++ b/src/data_factory_testing_framework/models/data_factory_element.py @@ -1,7 +1,7 @@ import json from typing import Any, Generic, TypeVar, Union -from data_factory_testing_framework.functions.expression_evaluator import ExpressionEvaluator +from data_factory_testing_framework.functions import ExpressionEvaluator from data_factory_testing_framework.state import RunState T = TypeVar("T") diff --git a/tests/unit/functions/test_expression_evaluator.py b/tests/unit/functions/test_expression_evaluator.py index 64c13df0..5bc6ab75 100644 --- a/tests/unit/functions/test_expression_evaluator.py +++ b/tests/unit/functions/test_expression_evaluator.py @@ -7,493 +7,16 @@ StateIterationItemNotSetError, ) from data_factory_testing_framework.exceptions.variable_not_found_error import VariableNotFoundError -from data_factory_testing_framework.functions.expression_evaluator import ExpressionEvaluator +from data_factory_testing_framework.functions import ExpressionEvaluator from data_factory_testing_framework.pythonnet.csharp_datetime import CSharpDateTime from data_factory_testing_framework.state.dependency_condition import DependencyCondition from data_factory_testing_framework.state.pipeline_run_state import PipelineRunState from data_factory_testing_framework.state.pipeline_run_variable import PipelineRunVariable from data_factory_testing_framework.state.run_parameter import RunParameter from data_factory_testing_framework.state.run_parameter_type import RunParameterType -from lark import Token, Tree from pytest import param as p -@pytest.mark.parametrize( - ["expression", "expected"], - [ - p("value", Tree(Token("RULE", "literal_evaluation"), [Token("LITERAL_LETTER", "value")]), id="string_literal"), - p( - " value ", - Tree(Token("RULE", "literal_evaluation"), [Token("LITERAL_LETTER", "value")]), - id="string_with_ws_literal", - marks=pytest.mark.skip(""), - ), - p("11", Tree(Token("RULE", "literal_evaluation"), [Token("LITERAL_INT", "11")]), id="integer_literal"), - p( - "@pipeline().parameters.parameter", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "parameters"), - Token("EXPRESSION_PARAMETER_NAME", "parameter"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="pipeline_parameters_reference", - ), - p( - "@pipeline().globalParameters.parameter", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "globalParameters"), - Token("EXPRESSION_PARAMETER_NAME", "parameter"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="pipeline_global_parameters_reference", - ), - p( - "@variables('variable')", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_variable_reference"), - [Token("EXPRESSION_VARIABLE_NAME", "'variable'")], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="variables_reference", - ), - p( - "@activity('activityName').output.outputName", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_activity_reference"), - [ - Token("EXPRESSION_ACTIVITY_NAME", "'activityName'"), - Tree( - Token("RULE", "expression_object_accessor"), - [Token("EXPRESSION_PARAMETER_NAME", "output")], - ), - ], - ), - Tree( - Token("RULE", "expression_object_accessor"), [Token("EXPRESSION_PARAMETER_NAME", "outputName")] - ), - ], - ), - id="activity_reference", - ), - p( - "@dataset().parameterName", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_dataset_reference"), - [Token("EXPRESSION_PARAMETER_NAME", "parameterName")], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="dataset_reference", - ), - p( - "@linkedService().parameterName", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_linked_service_reference"), - [Token("EXPRESSION_PARAMETER_NAME", "parameterName")], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="linked_service_reference", - ), - p( - "@item()", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree(Token("RULE", "expression_item_reference"), []), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="item_reference", - ), - p( - "@concat('a', 'b' )", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "concat"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree(Token("RULE", "expression_parameter"), [Token("EXPRESSION_STRING", "'a'")]), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Token("EXPRESSION_STRING", "'b'"), - Token("EXPRESSION_WS", " "), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="function_call", - ), - p( - "@concat('a', 'b' )", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "concat"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree(Token("RULE", "expression_parameter"), [Token("EXPRESSION_STRING", "'a'")]), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Token("EXPRESSION_STRING", "'b'"), - Token("EXPRESSION_WS", " "), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="function_call", - ), - p( - "@concat('https://example.com/jobs/', '123''', concat('&', 'abc,'))", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "concat"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_STRING", "'https://example.com/jobs/'")], - ), - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_WS", " "), Token("EXPRESSION_STRING", "'123'''")], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "concat"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_STRING", "'&'")], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Token("EXPRESSION_STRING", "'abc,'"), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="function_call_with_nested_function_and_single_quote", - ), - p( - "concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", - Tree( - Token("RULE", "literal_evaluation"), - [ - Token( - "LITERAL_LETTER", - "concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", - ) - ], - ), - id="literal_function_call_with_nested_function_and_single_quote", - ), - p( - "@concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "concat"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_STRING", "'https://example.com/jobs/'")], - ), - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_WS", " "), Token("EXPRESSION_STRING", "'123'''")], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_variable_reference"), - [Token("EXPRESSION_VARIABLE_NAME", "'abc'")], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - ], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "parameters"), - Token("EXPRESSION_PARAMETER_NAME", "abc"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - ], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_activity_reference"), - [ - Token("EXPRESSION_ACTIVITY_NAME", "'abc'"), - Tree( - Token("RULE", "expression_object_accessor"), - [Token("EXPRESSION_PARAMETER_NAME", "output")], - ), - ], - ), - Tree( - Token("RULE", "expression_object_accessor"), - [Token("EXPRESSION_PARAMETER_NAME", "abc")], - ), - ], - ), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - id="function_call_with_adf_native_functions", - ), - p( - "@createArray('a', createArray('a', 'b'))[1][1]", - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "createArray"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree(Token("RULE", "expression_parameter"), [Token("EXPRESSION_STRING", "'a'")]), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_function_call"), - [ - Token("EXPRESSION_FUNCTION_NAME", "createArray"), - Tree( - Token("RULE", "expression_function_parameters"), - [ - Tree( - Token("RULE", "expression_parameter"), - [Token("EXPRESSION_STRING", "'a'")], - ), - Tree( - Token("RULE", "expression_parameter"), - [ - Token("EXPRESSION_WS", " "), - Token("EXPRESSION_STRING", "'b'"), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - ], - ), - ], - ), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), [Token("EXPRESSION_ARRAY_INDEX", "[1]")]), - Tree(Token("RULE", "expression_object_accessor"), [Token("EXPRESSION_ARRAY_INDEX", "[1]")]), - ], - ), - id="function_call_with_nested_array_index", - ), - p( - "/repos/@{pipeline().globalParameters.OpsPrincipalClientId}/", - Tree( - Token("RULE", "literal_interpolation"), - [ - Token("LITERAL_LETTER", "/repos/"), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "globalParameters"), - Token("EXPRESSION_PARAMETER_NAME", "OpsPrincipalClientId"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - Token("LITERAL_LETTER", "/"), - ], - ), - id="string_interpolation", - ), - p( - "/repos/@{pipeline().globalParameters.OpsPrincipalClientId}/@{pipeline().parameters.SubPath}", - Tree( - Token("RULE", "literal_interpolation"), - [ - Token("LITERAL_LETTER", "/repos/"), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "globalParameters"), - Token("EXPRESSION_PARAMETER_NAME", "OpsPrincipalClientId"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - Token("LITERAL_LETTER", "/"), - Tree( - Token("RULE", "expression_evaluation"), - [ - Tree( - Token("RULE", "expression_pipeline_reference"), - [ - Token("EXPRESSION_PIPELINE_PROPERTY", "parameters"), - Token("EXPRESSION_PARAMETER_NAME", "SubPath"), - ], - ), - Tree(Token("RULE", "expression_object_accessor"), []), - ], - ), - ], - ), - id="string_interpolation_multiple_expressions", - ), - ], -) -def test_parse(expression: str, expected: Tree[Token]) -> None: - # Arrange - evaluator = ExpressionEvaluator() - - # Act - actual = evaluator.parse(expression) - - # Assert - assert actual == expected - - @pytest.mark.parametrize( ["expression", "state", "expected"], [ @@ -641,12 +164,43 @@ def test_parse(expression: str, expected: Tree[Token]) -> None: } } ), + # TODO: fix this "0.016666666666666666test", id="function_call_with_nested_property", marks=pytest.mark.xfail( reason="We do not support automatic type conversion yet. Here float is passed to concat (which expects str)." ), ), + p( + "concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", + PipelineRunState(), + "concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", + ), + p( + "@concat('https://example.com/jobs/', '123''', variables('abc'), pipeline().parameters.abc, activity('abc').output.abc)", + PipelineRunState( + variables=[PipelineRunVariable("abc", "defaultvalue_")], + parameters=[RunParameter(RunParameterType.Pipeline, "abc", "testvalue_02")], + pipeline_activity_results={"abc": {"output": {"abc": "_testvalue_01"}}}, + ), + "https://example.com/jobs/123'defaultvalue_testvalue_02_testvalue_01", + ), + p("@createArray('a', createArray('a', 'b'))[1][1]", PipelineRunState(), "b"), + p( + "/repos/@{pipeline().globalParameters.OpsPrincipalClientId}/", + PipelineRunState(parameters=[RunParameter(RunParameterType.Global, "OpsPrincipalClientId", "id")]), + "/repos/id/", + ), + p( + "/repos/@{pipeline().globalParameters.OpsPrincipalClientId}/@{pipeline().parameters.SubPath}", + PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Global, "OpsPrincipalClientId", "id"), + RunParameter(RunParameterType.Pipeline, "SubPath", "apath"), + ] + ), + "/repos/id/apath", + ), p( "@activity('Sample').output.billingReference.billableDuration[0].duration", PipelineRunState( @@ -1048,3 +602,154 @@ def test_json_nested_object_with_list_and_attributes(json_expression: str, acces actual = evaluator.evaluate(expression, state) assert actual == expected + + +@pytest.mark.parametrize( + ["logical_operator", "parameter_left", "parameter_right", "state", "expected"], + [ + p( + "or", + "aval", + "bval", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "aval")]), + True, + id="or_true_with_left_parameter_short_circuit", + ), + p( + "or", + "OTHER", + "bval", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "aval")]), + ParameterNotFoundError, + id="or_false_with_right_parameter_exception", + ), + p( + "or", + "OTHER", + "bval", + PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "a", "aval"), + RunParameter(RunParameterType.Pipeline, "b", "bval"), + ] + ), + True, + id="or_true_with_right_parameter", + ), + p( + "or", + "OTHER", + "OTHER", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "aval")]), + ParameterNotFoundError, + id="or_false_with_both_parameters_right_exception", + ), + p( + "or", + "OTHER", + "OTHER", + PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "a", "aval"), + RunParameter(RunParameterType.Pipeline, "b", "bval"), + ] + ), + False, + id="or_false_with_both_parameters", + ), + p( + "and", + "aval", + "bval", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "aval")]), + ParameterNotFoundError, + id="and_false_with_right_parameter_exception", + ), + p( + "and", + "OTHER", + "bval", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "aval")]), + False, + id="and_false_with_left_parameter_short_circuit", + ), + p( + "and", + "OTHER", + "bval", + PipelineRunState( + parameters=[ + RunParameter(RunParameterType.Pipeline, "a", "aval"), + RunParameter(RunParameterType.Pipeline, "b", "bval"), + ] + ), + False, + id="and_false_with_right_parameter", + ), + ], +) +def test_boolean_operators_short_circuit( + logical_operator: str, + parameter_left: str, + parameter_right: str, + state: PipelineRunState, + expected: Union[bool, Exception], +) -> None: + # Arrange + expression = f"@{logical_operator}(equals(pipeline().parameters.a,'{parameter_left}'),equals(pipeline().parameters.b,'{parameter_right}'))" + + evaluator = ExpressionEvaluator() + + # Act / Assert + if isinstance(expected, bool): + actual = evaluator.evaluate(expression, state) + assert actual == expected + else: + with pytest.raises(expected): + evaluator.evaluate(expression, state) + + +@pytest.mark.parametrize( + ["expression", "state", "expected"], + [ + p( + "@if(equals(pipeline().parameters.a, 'MATCH'), 'LEFT_BRANCH', 'RIGHT_BRANCH')", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "MATCH")]), + "LEFT_BRANCH", + id="if_true", + ), + p( + "@if(equals(pipeline().parameters.a, 'NO_MATCH'), 'LEFT_BRANCH', 'RIGHT_BRANCH')", + PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "a", "MATCH")]), + "RIGHT_BRANCH", + id="if_false", + ), + p( + "@if(equals('MATCH', 'MATCH'), pipeline().parameters.b, 'RIGHT_BRANCH')", + PipelineRunState(), + ParameterNotFoundError, + id="if_false_left_no_parameter", + ), + p( + "@if(equals('MATCH', 'MATCH'), 'LEFT_BRANCH', pipeline().parameters.b)", + PipelineRunState(), + "LEFT_BRANCH", + id="if_true_right_no_parameter", + ), + ], +) +def test_conditional_expression_with_branching( + expression: str, state: PipelineRunState, expected: Union[str, int, bool, float, Exception] +) -> None: + # Arrange + evaluator = ExpressionEvaluator() + + # Act / Assert + if isinstance(expected, (str, int, bool, float)): + actual = evaluator.evaluate(expression, state) + + # Assert + assert actual == expected + else: + with pytest.raises(expected): + evaluator.evaluate(expression, state) diff --git a/tests/unit/functions/test_functions_logical_implementation.py b/tests/unit/functions/test_functions_logical_implementation.py index f07cacc1..3a850ac9 100644 --- a/tests/unit/functions/test_functions_logical_implementation.py +++ b/tests/unit/functions/test_functions_logical_implementation.py @@ -5,23 +5,6 @@ from pytest import param -@pytest.mark.parametrize( - "expression1, expression_2, expected", - [ - (True, True, True), - (True, False, False), - (False, True, False), - (False, False, False), - ], -) -def test_and(expression1: bool, expression_2: bool, expected: bool) -> None: - # Act - actual = logical_functions.and_(expression1, expression_2) - - # Assert - assert actual == expected - - @pytest.mark.parametrize( "object1, object2, expected", [ @@ -68,21 +51,6 @@ def test_greater_or_equals(value: Any, compare_to: Any, expected: bool) -> None: assert actual == expected -@pytest.mark.parametrize( - ["expression", "value_if_true", "value_if_false", "expected"], - [ - param(True, "yes", "no", "yes"), - param(False, "yes", "no", "no"), - ], -) -def test_if(expression: bool, value_if_true: Any, value_if_false: Any, expected: Any) -> None: # noqa: ANN401 - # Act - actual = logical_functions.if_(expression, value_if_true, value_if_false) - - # Assert - assert actual == expected - - @pytest.mark.parametrize( ["value", "compare_to", "expected"], [ @@ -134,20 +102,3 @@ def test_not(expression: bool, expected: bool) -> None: # noqa: ANN401 # Assert assert actual == expected - - -@pytest.mark.parametrize( - ["expression1", "expression2", "expected"], - [ - param(True, True, True), - param(True, False, True), - param(False, True, True), - param(False, False, False), - ], -) -def test_or(expression1: bool, expression2: bool, expected: bool) -> None: # noqa: ANN401 - # Act - actual = logical_functions.or_(expression1, expression2) - - # Assert - assert actual == expected