diff --git a/src/AzureDataFactory.TestingFramework.sln b/src/AzureDataFactory.TestingFramework.sln index 7c086dca..a23d896c 100644 --- a/src/AzureDataFactory.TestingFramework.sln +++ b/src/AzureDataFactory.TestingFramework.sln @@ -1,10 +1,18 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework", "AzureDataFactory.TestingFramework\AzureDataFactory.TestingFramework.csproj", "{F5C4C98C-B717-4ED8-95EA-5DD51CB87C92}" +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34031.279 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework", "AzureDataFactory.TestingFramework\AzureDataFactory.TestingFramework.csproj", "{F5C4C98C-B717-4ED8-95EA-5DD51CB87C92}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework.Tests", "AzureDataFactory.TestingFramework.Tests\AzureDataFactory.TestingFramework.Tests.csproj", "{76089DF4-314B-4556-9ED7-D2B2F3BBA87A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework.Tests", "AzureDataFactory.TestingFramework.Tests\AzureDataFactory.TestingFramework.Tests.csproj", "{76089DF4-314B-4556-9ED7-D2B2F3BBA87A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureDataFactory.TestingFramework.Example", "AzureDataFactory.TestingFramework.Example\AzureDataFactory.TestingFramework.Example.csproj", "{F044EB13-DF36-43FB-90E8-9ADDE0E03821}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureDataFactory.TestingFramework.Example", "AzureDataFactory.TestingFramework.Example\AzureDataFactory.TestingFramework.Example.csproj", "{F044EB13-DF36-43FB-90E8-9ADDE0E03821}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{2EC1EFEA-98DC-47FC-9EAD-69541C65ACA1}" + ProjectSection(SolutionItems) = preProject + ..\README.md = ..\README.md + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -25,6 +33,7 @@ Global {F044EB13-DF36-43FB-90E8-9ADDE0E03821}.Release|Any CPU.ActiveCfg = Release|Any CPU {F044EB13-DF36-43FB-90E8-9ADDE0E03821}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection - GlobalSection(NestedProjects) = preSolution + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE EndGlobalSection EndGlobal diff --git a/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj b/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj index 5ad744eb..4867975e 100644 --- a/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj +++ b/src/AzureDataFactory.TestingFramework/AzureDataFactory.TestingFramework.csproj @@ -16,7 +16,7 @@ AzureDataFactory.TestingFramework - 0.1.3-alpha + 0.1.4-alpha arjendev true diff --git a/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs b/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs index 68854278..98235676 100644 --- a/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs +++ b/src/AzureDataFactory.TestingFramework/Expressions/ActivityExpression.cs @@ -19,7 +19,7 @@ public ActivityExpression(string expression) : base(expression) public TType Evaluate(PipelineRunState state) { var (activityName, fields) = GetActivityNameAndFields(); - var activity = state.PipelineActivityResults.SingleOrDefault(x => string.Equals(x.Name, activityName, StringComparison.CurrentCultureIgnoreCase)) ?? + var activity = state.PipelineActivityResults.LastOrDefault(x => string.Equals(x.Name, activityName, StringComparison.CurrentCultureIgnoreCase)) ?? throw new ActivityNotFoundException(activityName); if (activity.Status == null) throw new ActivityNotEvaluatedException(activity.Name); diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs index a9ce1ec5..85442571 100644 --- a/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs +++ b/src/AzureDataFactory.TestingFramework/Functions/FunctionArgument.cs @@ -48,6 +48,7 @@ public TType Evaluate(RunState state) { { } type when type == typeof(bool) && bool.TryParse(evalExpression, out var boolValue) => (TType)(object) boolValue, { } type when type == typeof(int) && int.TryParse(evalExpression, out var intValue) => (TType)(object) intValue, + { } type when type == typeof(long) && long.TryParse(evalExpression, out var longValue) => (TType)(object) longValue, { } type when type == typeof(string) => (TType)(object) evalExpression.TrimOneChar('\''), { } type => throw new ArgumentException($"The result {evalExpression} with DataType: {type} could not be parsed accordingly.") }; diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs index 3b05d031..3cb02080 100644 --- a/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs +++ b/src/AzureDataFactory.TestingFramework/Functions/FunctionCall.cs @@ -41,6 +41,7 @@ public TType Evaluate(RunState state) { } when parameterType == typeof(float) => argument.Evaluate(state), { } when parameterType == typeof(double) => argument.Evaluate(state), { } when parameterType == typeof(object) => argument.Evaluate(state), + { IsArray: true } => argument.Evaluate(state), _ => throw new Exception($"Unsupported parameter type: {parameterType}") }; }).ToList(); diff --git a/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs b/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs index 7a715bf2..3b2a74a9 100644 --- a/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs +++ b/src/AzureDataFactory.TestingFramework/Functions/FunctionsRepository.cs @@ -15,8 +15,16 @@ public static class FunctionsRepository { "replace", (string input, string pattern, string replacement) => input.Replace(pattern, replacement) }, { "string", (string input) => input }, { "union" , (string arg0, string arg1) => JsonSerializer.Serialize(JsonSerializer.Deserialize(arg0).Union(JsonSerializer.Deserialize((arg1)))) }, - { "coalesce", (IEnumerable args) => args.FirstOrDefault(arg => !string.IsNullOrEmpty(arg)) } - + { "coalesce", (IEnumerable args) => args.FirstOrDefault(arg => !string.IsNullOrEmpty(arg)) }, + { "or", (bool a, bool b) => a || b }, + { "utcnow", () => DateTime.UtcNow.ToString("o") }, + { "utcNow", () => DateTime.UtcNow.ToString("o") }, + { "ticks", (string dateTime) => DateTime.Parse(dateTime).Ticks }, + { "sub", (long a, long b) => a - b }, + { "div", (long a, long b) => a / b }, + { "greaterOrEquals", (long a, long b) => a >= b }, + { "not", (bool value) => !value }, + { "empty", (object[] array) => array.Length == 0 }, }; public static void Register(string functionName, Delegate function) diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs index 25ee7c4a..3a3b95fc 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/ActivitiesEvaluator.cs @@ -6,26 +6,31 @@ public static class ActivitiesEvaluator { public static IEnumerable Evaluate(List activities, PipelineRunState state) { - while (state.PipelineActivityResults.Count != activities.Count) + while (state.ScopedPipelineActivityResults.Count != activities.Count) { var anyActivityEvaluated = false; foreach (var activity in activities - .Where(activity => !state.PipelineActivityResults.Contains(activity)) + .Where(activity => !state.ScopedPipelineActivityResults.Contains(activity)) .Where(activity => activity.AreDependencyConditionMet(state))) { - yield return (PipelineActivity) activity.Evaluate(state); + var evaluatedActivity = (PipelineActivity) activity.Evaluate(state); + if (evaluatedActivity is not IIterationActivity) + yield return evaluatedActivity; + anyActivityEvaluated = true; state.AddActivityResult(activity); - if (activity is ControlActivity controlActivity) + if (activity is IIterationActivity) { - if (controlActivity is UntilActivity untilActivity) + if (activity is UntilActivity untilActivity) { - while (untilActivity.Expression.Evaluate(state)) + do + { foreach (var child in untilActivity.EvaluateChildActivities(state)) yield return child; + } while (!untilActivity.Expression.Evaluate(state)); } - else + else if (activity is ControlActivity controlActivity) { foreach (var child in controlActivity.EvaluateChildActivities(state)) yield return child; diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs index 4a6b61f9..8bf5383b 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/Base/PipelineActivity.cs @@ -28,7 +28,7 @@ public bool AreDependencyConditionMet(PipelineRunState state) { foreach (var dependency in DependsOn) { - var dependencyActivity = state.PipelineActivityResults.SingleOrDefault(a => a.Name == dependency.Activity); + var dependencyActivity = state.ScopedPipelineActivityResults.SingleOrDefault(a => a.Name == dependency.Activity); // If dependency is not yet evaluated, conditions are not met if (dependencyActivity == null) diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs index 2df8be85..bfe87618 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ControlActivity.cs @@ -19,5 +19,7 @@ public virtual IEnumerable EvaluateChildActivities(PipelineRun { yield return activity; } + + state.AddScopedActivityResultsFromScopedState(scopedState); } } \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs index 3d95f050..9240e782 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/ForEachActivity.cs @@ -4,7 +4,7 @@ namespace AzureDataFactory.TestingFramework.Models; -public partial class ForEachActivity +public partial class ForEachActivity : IIterationActivity { protected override List GetNextActivities() { diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IIterationActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IIterationActivity.cs new file mode 100644 index 00000000..015a365f --- /dev/null +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IIterationActivity.cs @@ -0,0 +1,6 @@ +namespace AzureDataFactory.TestingFramework.Models; + +public interface IIterationActivity +{ + +} \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs index 04be8bf1..e3e34b60 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/IfConditionActivity.cs @@ -4,14 +4,14 @@ namespace AzureDataFactory.TestingFramework.Models; -public partial class IfConditionActivity +public partial class IfConditionActivity : IIterationActivity { protected override List GetNextActivities() { - return _evaluatedExpression.Value ? IfTrueActivities.ToList() : IfFalseActivities.ToList(); + return EvaluatedExpression ? IfTrueActivities.ToList() : IfFalseActivities.ToList(); } - public bool? _evaluatedExpression; + private bool? _evaluatedExpression; public bool EvaluatedExpression => _evaluatedExpression ?? throw new InvalidOperationException("Expression has not been evaluated yet."); public override DataFactoryEntity Evaluate(PipelineRunState state) { diff --git a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs index 95e2e80e..09ded435 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Activities/ControlActivity/UntilActivity.cs @@ -3,20 +3,10 @@ namespace AzureDataFactory.TestingFramework.Models; -public partial class UntilActivity +public partial class UntilActivity : IIterationActivity { protected override List GetNextActivities() { return Activities.ToList(); } - - private bool? _evaluatedExpression; - public override PipelineActivity Evaluate(PipelineRunState state) - { - base.Evaluate(state); - - _evaluatedExpression = Expression.Evaluate(state); - - return this; - } } \ No newline at end of file diff --git a/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs b/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs index c8431870..c5cb0532 100644 --- a/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs +++ b/src/AzureDataFactory.TestingFramework/Models/Pipelines/PipelineRunState.cs @@ -9,6 +9,7 @@ public class PipelineRunState : RunState public List Variables { get; } public string? IterationItem { get; set; } public List PipelineActivityResults { get; } + public List ScopedPipelineActivityResults { get; } public PipelineRunState(List parameters, IDictionary variables) : base(parameters) { @@ -27,13 +28,16 @@ public PipelineRunState(List parameters, IDictionary(); + ScopedPipelineActivityResults = new List(); IterationItem = null; } - public PipelineRunState(List parameters, List variables, string? iterationItem) : base(parameters) + public PipelineRunState(List parameters, List variables, List activityResults, string? iterationItem) : base(parameters) { Variables = variables; PipelineActivityResults = new List(); + PipelineActivityResults.AddRange(activityResults); + ScopedPipelineActivityResults = new List(); IterationItem = iterationItem; } @@ -41,16 +45,23 @@ public PipelineRunState(): base(new List()) { Variables = new List(); PipelineActivityResults = new List(); + ScopedPipelineActivityResults = new List(); IterationItem = null; } public void AddActivityResult(IPipelineActivityResult pipelineActivityResult) { + ScopedPipelineActivityResults.Add(pipelineActivityResult); PipelineActivityResults.Add(pipelineActivityResult); } + public void AddScopedActivityResultsFromScopedState(PipelineRunState scopedState) + { + PipelineActivityResults.AddRange(scopedState.ScopedPipelineActivityResults); + } + public PipelineRunState CreateIterationScope(string? iterationItem) { - return new PipelineRunState(Parameters, Variables, iterationItem); + return new PipelineRunState(Parameters, Variables, PipelineActivityResults, iterationItem); } } \ No newline at end of file