diff --git a/README.md b/README.md index a937c0f0..ab3db9ae 100644 --- a/README.md +++ b/README.md @@ -71,60 +71,62 @@ The samples seen below is the _only_ code that you need to write! The framework # Assert assert "https://example.com/jobs" == activity.type_properties["url"].value assert "POST" == activity.type_properties["method"].value - assert "{ \n \"JobId\": \"123\",\n \"JobName\": \"Job-123\",\n \"Version\": \"version1\",\n}" == activity.type_properties["body"].value - ``` + body = activity.type_properties["body"].get_json_value() + assert "123" == body["JobId"] + assert "Job-123" == body["JobName"] + assert "version1" == body["Version"] + ``` -2. Evaluate Pipelines and test the flow of activities given a specific input - - ```python - # Arrange - pipeline: PipelineResource = test_framework.repository.get_pipeline_by_name("batch_job") - - # Runs the pipeline with the provided parameters - activities = test_framework.evaluate_pipeline(pipeline, [ - RunParameter(RunParameterType.Pipeline, "JobId", "123"), - RunParameter(RunParameterType.Pipeline, "ContainerName", "test-container"), - RunParameter(RunParameterType.Global, "BaseUrl", "https://example.com"), - ]) - - set_variable_activity: Activity = next(activities) - assert set_variable_activity is not None - assert "Set JobName" == set_variable_activity.name - assert "JobName" == activity.type_properties["variableName"] - assert "Job-123" == activity.type_properties["value"].value - - get_version_activity = next(activities) - assert get_version_activity is not None - assert "Get version" == get_version_activity.name - assert "https://example.com/version" == get_version_activity.type_properties["url"].value - assert "GET" == get_version_activity.type_properties["method"] - get_version_activity.set_result(DependencyCondition.Succeeded,{"Version": "version1"}) - - create_batch_activity = next(activities) - assert create_batch_activity is not None - assert "Trigger Azure Batch Job" == create_batch_activity.name - assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].value - assert "POST" == create_batch_activity.type_properties["method"] - assert "{ \n \"JobId\": \"123\",\n \"JobName\": \"Job-123\",\n \"Version\": \"version1\",\n}" == create_batch_activity.type_properties["body"].value - - with pytest.raises(StopIteration): - next(activities) - ``` + 2. Evaluate Pipelines and test the flow of activities given a specific input + + ```python + # Arrange + pipeline: PipelineResource = test_framework.repository.get_pipeline_by_name("batch_job") + + # Runs the pipeline with the provided parameters + activities = test_framework.evaluate_pipeline(pipeline, [ + RunParameter(RunParameterType.Pipeline, "JobId", "123"), + RunParameter(RunParameterType.Pipeline, "ContainerName", "test-container"), + RunParameter(RunParameterType.Global, "BaseUrl", "https://example.com"), + ]) + + set_variable_activity: Activity = next(activities) + assert set_variable_activity is not None + assert "Set JobName" == set_variable_activity.name + assert "JobName" == activity.type_properties["variableName"] + assert "Job-123" == activity.type_properties["value"].value + + get_version_activity = next(activities) + assert get_version_activity is not None + assert "Get version" == get_version_activity.name + assert "https://example.com/version" == get_version_activity.type_properties["url"].value + assert "GET" == get_version_activity.type_properties["method"] + get_version_activity.set_result(DependencyCondition.Succeeded,{"Version": "version1"}) + + create_batch_activity = next(activities) + assert create_batch_activity is not None + assert "Trigger Azure Batch Job" == create_batch_activity.name + assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].value + assert "POST" == create_batch_activity.type_properties["method"] + body = create_batch_activity.type_properties["body"].get_json_value() + assert "123" == body["JobId"] + assert "Job-123" == body["JobName"] + assert "version1" == body["Version"] + + with pytest.raises(StopIteration): + next(activities) + ``` > See Examples folder for more samples ## Registering missing expression functions -As the framework is interpreting expressions containing functions, these functions need to be implemented in Python. The goal is to start supporting more and more functions, but if a function is not supported, then the following code can be used to register a missing function: +As the framework is interpreting expressions containing functions, these functions are implemented in Python. All functions are implemented, but there may be bugs in some of them. You can override their implementation through: ```python FunctionsRepository.register("concat", lambda arguments: "".join(arguments)) FunctionsRepository.register("trim", lambda text, trim_argument: text.strip(trim_argument[0])) -``` - -On runtime when evaluating expressions, the framework will try to find a matching function and assert the expected amount of arguments are supplied. If no matching function is found, then an exception will be thrown. - -> Feel free to add a pull request with your own custom functions, so that they can be added to the framework and enjoyed by everyone. +``` ## Tips diff --git a/examples/data_factory/batch_job/test_data_factory_batchjob_functional.py b/examples/data_factory/batch_job/test_data_factory_batchjob_functional.py index c8b76343..d1710e47 100644 --- a/examples/data_factory/batch_job/test_data_factory_batchjob_functional.py +++ b/examples/data_factory/batch_job/test_data_factory_batchjob_functional.py @@ -81,43 +81,30 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None: activity = next(activities) assert activity.name == "Set CommonEnvironmentSettings" assert activity.type_properties["variableName"] == "CommonEnvironmentSettings" - # noqa: E501 + + common_environment_settings = activity.type_properties["value"].get_json_value() + assert len(common_environment_settings) == 8 + assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert common_environment_settings[0]["value"] == "test-application" + assert common_environment_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert common_environment_settings[1]["value"] == "1.5.0" + assert common_environment_settings[2]["name"] == "MANAGER_APP_PACKAGE" + assert common_environment_settings[2]["value"] == "batchmanager" + assert common_environment_settings[3]["name"] == "MANAGER_APP_PACKAGE_VERSION" + assert common_environment_settings[3]["value"] == "2.0.0" + assert common_environment_settings[4]["name"] == "BATCH_JOB_TIMEOUT" + assert common_environment_settings[4]["value"] == "PT4H" + assert common_environment_settings[5]["name"] == "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME" + assert common_environment_settings[5]["value"] == "batch-account-name" + assert common_environment_settings[6]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID" assert ( - activity.type_properties["value"].value - == """[ - { - "name": "WORKLOAD_APP_PACKAGE", - "value": "test-application" - }, - { - "name": "WORKLOAD_APP_PACKAGE_VERSION", - "value": "1.5.0" - }, - { - "name": "MANAGER_APP_PACKAGE", - "value": "batchmanager" - }, - { - "name": "MANAGER_APP_PACKAGE_VERSION", - "value": "2.0.0" - }, - { - "name": "BATCH_JOB_TIMEOUT", - "value": "PT4H" - }, - { - "name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", - "value": "batch-account-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", - "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", - "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - } - ]""" # noqa: E501 + common_environment_settings[6]["value"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 + ) + assert common_environment_settings[7]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID" + assert ( + common_environment_settings[7]["value"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 ) activity = next(activities) @@ -149,72 +136,57 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None: == "https://batch-account-name.westeurope.batch.azure.com/jobs?api-version=2022-10-01.16.0" ) assert activity.type_properties["method"] == "POST" + + body = activity.type_properties["body"].get_json_value() + assert body["id"] == "802100a5-ec79-4a52-be62-8d6109f3ff9a" + assert body["priority"] == 100 + assert body["constraints"]["maxWallClockTime"] == "PT4H" + assert body["constraints"]["maxTaskRetryCount"] == 0 + job_manager_task = body["jobManagerTask"] + assert job_manager_task["id"] == "Manager" + assert job_manager_task["displayName"] == "Manager" + assert job_manager_task["authenticationTokenSettings"]["access"] == ["job"] assert ( - activity.type_properties["body"].value - == """{ - "id": "802100a5-ec79-4a52-be62-8d6109f3ff9a", - "priority": 100, - "constraints": { - "maxWallClockTime":"PT4H", - "maxTaskRetryCount": 0 - }, - "jobManagerTask": { - "id": "Manager", - "displayName": "Manager", - "authenticationTokenSettings": { - "access": [ - "job" - ] - }, - "commandLine": "/bin/bash -c \\"python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz && python3 -m test-application job --parameter1 dummy --parameter2 another-dummy\\"", - "applicationPackageReferences": [ - { - "applicationId": "batchmanager", - "version": "2.0.0" - }, - { - "applicationId": "test-application", - "version": "1.5.0" - } - ], - "outputFiles": [ - { - "destination": { - "container": { - "containerUrl": "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a", - "identityReference": { - "resourceId": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - }, - "path": "Manager/$TaskLog" - } - }, - "filePattern": "../*.txt", - "uploadOptions": { - "uploadCondition": "taskcompletion" - } - } - ], - "environmentSettings": [], - "requiredSlots": 1, - "killJobOnCompletion": false, - "userIdentity": { - "username": null, - "autoUser": { - "scope": "pool", - "elevationLevel": "nonadmin" - } - }, - "runExclusive": true, - "allowLowPriorityNode": true - }, - "poolInfo": { - "poolId": "batch-pool-id" - }, - "onAllTasksComplete": "terminatejob", - "onTaskFailure": "noaction", - "usesTaskDependencies": true, - "commonEnvironmentSettings": [{"name": "WORKLOAD_APP_PACKAGE", "value": "test-application"}, {"name": "WORKLOAD_APP_PACKAGE_VERSION", "value": "1.5.0"}, {"name": "MANAGER_APP_PACKAGE", "value": "batchmanager"}, {"name": "MANAGER_APP_PACKAGE_VERSION", "value": "2.0.0"}, {"name": "BATCH_JOB_TIMEOUT", "value": "PT4H"}, {"name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", "value": "batch-account-name"}, {"name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name"}, {"name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name"}]}""" # noqa: E501 + job_manager_task["commandLine"] + == '/bin/bash -c "python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz && python3 -m test-application job --parameter1 dummy --parameter2 another-dummy"' # noqa: E501 + ) + application_package_references = job_manager_task["applicationPackageReferences"] + assert len(application_package_references) == 2 + assert application_package_references[0]["applicationId"] == "batchmanager" + assert application_package_references[0]["version"] == "2.0.0" + assert application_package_references[1]["applicationId"] == "test-application" + assert application_package_references[1]["version"] == "1.5.0" + output_files = job_manager_task["outputFiles"] + assert len(output_files) == 1 + assert ( + output_files[0]["destination"]["container"]["containerUrl"] + == "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a" + ) + assert ( + output_files[0]["destination"]["container"]["identityReference"]["resourceId"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 ) + assert output_files[0]["destination"]["container"]["path"] == "Manager/$TaskLog" + assert output_files[0]["filePattern"] == "../*.txt" + assert output_files[0]["uploadOptions"]["uploadCondition"] == "taskcompletion" + assert len(job_manager_task["environmentSettings"]) == 0 + assert job_manager_task["requiredSlots"] == 1 + assert job_manager_task["killJobOnCompletion"] is False + assert job_manager_task["userIdentity"]["username"] is None + assert job_manager_task["userIdentity"]["autoUser"]["scope"] == "pool" + assert job_manager_task["userIdentity"]["autoUser"]["elevationLevel"] == "nonadmin" + assert job_manager_task["runExclusive"] is True + assert job_manager_task["allowLowPriorityNode"] is True + assert body["poolInfo"]["poolId"] == "batch-pool-id" + assert body["onAllTasksComplete"] == "terminatejob" + assert body["onTaskFailure"] == "noaction" + assert body["usesTaskDependencies"] is True + assert len(body["commonEnvironmentSettings"]) == 8 + common_environment_settings = body["commonEnvironmentSettings"] + assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert common_environment_settings[0]["value"] == "test-application" + assert common_environment_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert common_environment_settings[1]["value"] == "1.5.0" activity = next(activities) assert activity.name == "Monitor Batch Job" diff --git a/examples/data_factory/batch_job/test_data_factory_batchjob_unit.py b/examples/data_factory/batch_job/test_data_factory_batchjob_unit.py index 61277005..60cdd708 100644 --- a/examples/data_factory/batch_job/test_data_factory_batchjob_unit.py +++ b/examples/data_factory/batch_job/test_data_factory_batchjob_unit.py @@ -155,42 +155,26 @@ def test_set_common_environment_settings(test_framework: TestFramework, pipeline activity.evaluate(state) # Assert - expected_settings = """[ - { - "name": "WORKLOAD_APP_PACKAGE", - "value": "workload" - }, - { - "name": "WORKLOAD_APP_PACKAGE_VERSION", - "value": "0.13.2" - }, - { - "name": "MANAGER_APP_PACKAGE", - "value": "managerworkload" - }, - { - "name": "MANAGER_APP_PACKAGE_VERSION", - "value": "0.13.2" - }, - { - "name": "BATCH_JOB_TIMEOUT", - "value": "PT4H" - }, - { - "name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", - "value": "batch-account-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", - "value": "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", - "value": "workload-user-assigned-identity-client-id" - } - ]""" - assert activity.type_properties["value"].value == expected_settings - assert state.get_variable_by_name("CommonEnvironmentSettings").value == expected_settings + env_settings = activity.type_properties["value"].get_json_value() + assert env_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert env_settings[0]["value"] == "workload" + assert env_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert env_settings[1]["value"] == "0.13.2" + assert env_settings[2]["name"] == "MANAGER_APP_PACKAGE" + assert env_settings[2]["value"] == "managerworkload" + assert env_settings[3]["name"] == "MANAGER_APP_PACKAGE_VERSION" + assert env_settings[3]["value"] == "0.13.2" + assert env_settings[4]["name"] == "BATCH_JOB_TIMEOUT" + assert env_settings[4]["value"] == "PT4H" + assert env_settings[5]["name"] == "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME" + assert env_settings[5]["value"] == "batch-account-name" + assert env_settings[6]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID" + assert ( + env_settings[6]["value"] + == "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" + ) + assert env_settings[7]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID" + assert env_settings[7]["value"] == "workload-user-assigned-identity-client-id" def test_create_job_storage_container(test_framework: TestFramework, pipeline: Pipeline) -> None: @@ -313,71 +297,52 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) - ) assert "POST" == activity.type_properties["method"] - expected_body = """{ - "id": "8b6b545b-c583-4a06-adf7-19ff41370aba", - "priority": 100, - "constraints": { - "maxWallClockTime":"PT4H", - "maxTaskRetryCount": 0 - }, - "jobManagerTask": { - "id": "Manager", - "displayName": "Manager", - "authenticationTokenSettings": { - "access": [ - "job" - ] - }, - "commandLine": "/bin/bash -c \\"python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_managerworkload_0_13_2/managerworkload.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz && python3 -m test-application-name job --parameter1 dummy --parameter2 another-dummy\\"", - "applicationPackageReferences": [ - { - "applicationId": "batchmanager", - "version": "2.0.0" - }, - { - "applicationId": "test-application-name", - "version": "1.5.0" - } - ], - "outputFiles": [ - { - "destination": { - "container": { - "containerUrl": "https://batchstorage.blob.core.windows.net/job-8b6b545b-c583-4a06-adf7-19ff41370aba", - "identityReference": { - "resourceId": "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" - }, - "path": "Manager/$TaskLog" - } - }, - "filePattern": "../*.txt", - "uploadOptions": { - "uploadCondition": "taskcompletion" - } - } - ], - "environmentSettings": [], - "requiredSlots": 1, - "killJobOnCompletion": false, - "userIdentity": { - "username": null, - "autoUser": { - "scope": "pool", - "elevationLevel": "nonadmin" - } - }, - "runExclusive": true, - "allowLowPriorityNode": true - }, - "poolInfo": { - "poolId": "test-application-batch-pool-id" - }, - "onAllTasksComplete": "terminatejob", - "onTaskFailure": "noaction", - "usesTaskDependencies": true, - "commonEnvironmentSettings": [{"name": "COMMON_ENV_SETTING", "value": "dummy"}, {"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}]}""" - - assert activity.type_properties["body"].value == expected_body + body = activity.type_properties["body"].get_json_value() + assert body["id"] == "8b6b545b-c583-4a06-adf7-19ff41370aba" + assert body["priority"] == 100 + assert body["constraints"]["maxWallClockTime"] == "PT4H" + assert body["constraints"]["maxTaskRetryCount"] == 0 + job_manager_task = body["jobManagerTask"] + assert job_manager_task["id"] == "Manager" + assert job_manager_task["displayName"] == "Manager" + assert job_manager_task["authenticationTokenSettings"]["access"] == ["job"] + assert ( + job_manager_task["commandLine"] + == '/bin/bash -c "python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_managerworkload_0_13_2/managerworkload.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz && python3 -m test-application-name job --parameter1 dummy --parameter2 another-dummy"' + ) + application_package_references = job_manager_task["applicationPackageReferences"] + assert application_package_references[0]["applicationId"] == "batchmanager" + assert application_package_references[0]["version"] == "2.0.0" + assert application_package_references[1]["applicationId"] == "test-application-name" + assert application_package_references[1]["version"] == "1.5.0" + assert ( + job_manager_task["outputFiles"][0]["destination"]["container"]["containerUrl"] + == "https://batchstorage.blob.core.windows.net/job-8b6b545b-c583-4a06-adf7-19ff41370aba" + ) + assert ( + job_manager_task["outputFiles"][0]["destination"]["container"]["identityReference"]["resourceId"] + == "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" # noqa: E501 + ) + assert job_manager_task["outputFiles"][0]["destination"]["container"]["path"] == "Manager/$TaskLog" + assert job_manager_task["outputFiles"][0]["filePattern"] == "../*.txt" + assert job_manager_task["outputFiles"][0]["uploadOptions"]["uploadCondition"] == "taskcompletion" + assert job_manager_task["environmentSettings"] == [] + assert job_manager_task["requiredSlots"] == 1 + assert job_manager_task["killJobOnCompletion"] is False + assert job_manager_task["userIdentity"]["username"] is None + assert job_manager_task["userIdentity"]["autoUser"]["scope"] == "pool" + assert job_manager_task["userIdentity"]["autoUser"]["elevationLevel"] == "nonadmin" + assert job_manager_task["runExclusive"] is True + assert job_manager_task["allowLowPriorityNode"] is True + assert body["poolInfo"]["poolId"] == "test-application-batch-pool-id" + assert body["onAllTasksComplete"] == "terminatejob" + assert body["onTaskFailure"] == "noaction" + assert body["usesTaskDependencies"] is True + common_environment_settings = body["commonEnvironmentSettings"] + assert common_environment_settings[0]["name"] == "COMMON_ENV_SETTING" + assert common_environment_settings[0]["value"] == "dummy" + assert common_environment_settings[1]["name"] == "STORAGE_ACCOUNT_NAME" + assert common_environment_settings[1]["value"] == "teststorage" def test_monitor_job(test_framework: TestFramework, pipeline: Pipeline) -> None: diff --git a/examples/fabric/batch_job/test_fabric_batchjob_functional.py b/examples/fabric/batch_job/test_fabric_batchjob_functional.py index 1afae13b..e3fcdd58 100644 --- a/examples/fabric/batch_job/test_fabric_batchjob_functional.py +++ b/examples/fabric/batch_job/test_fabric_batchjob_functional.py @@ -81,43 +81,30 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None: activity = next(activities) assert activity.name == "Set CommonEnvironmentSettings" assert activity.type_properties["variableName"] == "CommonEnvironmentSettings" - # noqa: E501 + + common_environment_settings = activity.type_properties["value"].get_json_value() + assert len(common_environment_settings) == 8 + assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert common_environment_settings[0]["value"] == "test-application" + assert common_environment_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert common_environment_settings[1]["value"] == "1.5.0" + assert common_environment_settings[2]["name"] == "MANAGER_APP_PACKAGE" + assert common_environment_settings[2]["value"] == "batchmanager" + assert common_environment_settings[3]["name"] == "MANAGER_APP_PACKAGE_VERSION" + assert common_environment_settings[3]["value"] == "2.0.0" + assert common_environment_settings[4]["name"] == "BATCH_JOB_TIMEOUT" + assert common_environment_settings[4]["value"] == "PT4H" + assert common_environment_settings[5]["name"] == "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME" + assert common_environment_settings[5]["value"] == "batch-account-name" + assert common_environment_settings[6]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID" assert ( - activity.type_properties["value"].value - == """[ - { - "name": "WORKLOAD_APP_PACKAGE", - "value": "test-application" - }, - { - "name": "WORKLOAD_APP_PACKAGE_VERSION", - "value": "1.5.0" - }, - { - "name": "MANAGER_APP_PACKAGE", - "value": "batchmanager" - }, - { - "name": "MANAGER_APP_PACKAGE_VERSION", - "value": "2.0.0" - }, - { - "name": "BATCH_JOB_TIMEOUT", - "value": "PT4H" - }, - { - "name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", - "value": "batch-account-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", - "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", - "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - } - ]""" # noqa: E501 + common_environment_settings[6]["value"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 + ) + assert common_environment_settings[7]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID" + assert ( + common_environment_settings[7]["value"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 ) activity = next(activities) @@ -146,72 +133,57 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None: assert activity.name == "Start Job" assert activity.type_properties["relativeUrl"] == "/jobs?api-version=2022-10-01.16.0" assert activity.type_properties["method"] == "POST" + + body = activity.type_properties["body"].get_json_value() + assert body["id"] == "802100a5-ec79-4a52-be62-8d6109f3ff9a" + assert body["priority"] == 100 + assert body["constraints"]["maxWallClockTime"] == "PT4H" + assert body["constraints"]["maxTaskRetryCount"] == 0 + job_manager_task = body["jobManagerTask"] + assert job_manager_task["id"] == "Manager" + assert job_manager_task["displayName"] == "Manager" + assert job_manager_task["authenticationTokenSettings"]["access"] == ["job"] assert ( - activity.type_properties["body"].value - == """{ - "id": "802100a5-ec79-4a52-be62-8d6109f3ff9a", - "priority": 100, - "constraints": { - "maxWallClockTime":"PT4H", - "maxTaskRetryCount": 0 - }, - "jobManagerTask": { - "id": "Manager", - "displayName": "Manager", - "authenticationTokenSettings": { - "access": [ - "job" - ] - }, - "commandLine": "/bin/bash -c \\"python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz && python3 -m test-application job --parameter1 dummy --parameter2 another-dummy\\"", - "applicationPackageReferences": [ - { - "applicationId": "batchmanager", - "version": "2.0.0" - }, - { - "applicationId": "test-application", - "version": "1.5.0" - } - ], - "outputFiles": [ - { - "destination": { - "container": { - "containerUrl": "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a", - "identityReference": { - "resourceId": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" - }, - "path": "Manager/$TaskLog" - } - }, - "filePattern": "../*.txt", - "uploadOptions": { - "uploadCondition": "taskcompletion" - } - } - ], - "environmentSettings": [], - "requiredSlots": 1, - "killJobOnCompletion": false, - "userIdentity": { - "username": null, - "autoUser": { - "scope": "pool", - "elevationLevel": "nonadmin" - } - }, - "runExclusive": true, - "allowLowPriorityNode": true - }, - "poolInfo": { - "poolId": "batch-pool-id" - }, - "onAllTasksComplete": "terminatejob", - "onTaskFailure": "noaction", - "usesTaskDependencies": true, - "commonEnvironmentSettings": [{"name": "WORKLOAD_APP_PACKAGE", "value": "test-application"}, {"name": "WORKLOAD_APP_PACKAGE_VERSION", "value": "1.5.0"}, {"name": "MANAGER_APP_PACKAGE", "value": "batchmanager"}, {"name": "MANAGER_APP_PACKAGE_VERSION", "value": "2.0.0"}, {"name": "BATCH_JOB_TIMEOUT", "value": "PT4H"}, {"name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", "value": "batch-account-name"}, {"name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name"}, {"name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", "value": "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name"}]}""" # noqa: E501 + job_manager_task["commandLine"] + == '/bin/bash -c "python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz && python3 -m test-application job --parameter1 dummy --parameter2 another-dummy"' # noqa: E501 + ) + application_package_references = job_manager_task["applicationPackageReferences"] + assert len(application_package_references) == 2 + assert application_package_references[0]["applicationId"] == "batchmanager" + assert application_package_references[0]["version"] == "2.0.0" + assert application_package_references[1]["applicationId"] == "test-application" + assert application_package_references[1]["version"] == "1.5.0" + output_files = job_manager_task["outputFiles"] + assert len(output_files) == 1 + assert ( + output_files[0]["destination"]["container"]["containerUrl"] + == "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a" + ) + assert ( + output_files[0]["destination"]["container"]["identityReference"]["resourceId"] + == "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501 ) + assert output_files[0]["destination"]["container"]["path"] == "Manager/$TaskLog" + assert output_files[0]["filePattern"] == "../*.txt" + assert output_files[0]["uploadOptions"]["uploadCondition"] == "taskcompletion" + assert len(job_manager_task["environmentSettings"]) == 0 + assert job_manager_task["requiredSlots"] == 1 + assert job_manager_task["killJobOnCompletion"] is False + assert job_manager_task["userIdentity"]["username"] is None + assert job_manager_task["userIdentity"]["autoUser"]["scope"] == "pool" + assert job_manager_task["userIdentity"]["autoUser"]["elevationLevel"] == "nonadmin" + assert job_manager_task["runExclusive"] is True + assert job_manager_task["allowLowPriorityNode"] is True + assert body["poolInfo"]["poolId"] == "batch-pool-id" + assert body["onAllTasksComplete"] == "terminatejob" + assert body["onTaskFailure"] == "noaction" + assert body["usesTaskDependencies"] is True + assert len(body["commonEnvironmentSettings"]) == 8 + common_environment_settings = body["commonEnvironmentSettings"] + assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert common_environment_settings[0]["value"] == "test-application" + assert common_environment_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert common_environment_settings[1]["value"] == "1.5.0" activity = next(activities) assert activity.name == "Monitor Batch Job" diff --git a/examples/fabric/batch_job/test_fabric_batchjob_unit.py b/examples/fabric/batch_job/test_fabric_batchjob_unit.py index 6683c5dd..79c32196 100644 --- a/examples/fabric/batch_job/test_fabric_batchjob_unit.py +++ b/examples/fabric/batch_job/test_fabric_batchjob_unit.py @@ -156,42 +156,26 @@ def test_set_common_environment_settings(test_framework: TestFramework, pipeline activity.evaluate(state) # Assert - expected_settings = """[ - { - "name": "WORKLOAD_APP_PACKAGE", - "value": "workload" - }, - { - "name": "WORKLOAD_APP_PACKAGE_VERSION", - "value": "0.13.2" - }, - { - "name": "MANAGER_APP_PACKAGE", - "value": "managerworkload" - }, - { - "name": "MANAGER_APP_PACKAGE_VERSION", - "value": "0.13.2" - }, - { - "name": "BATCH_JOB_TIMEOUT", - "value": "PT4H" - }, - { - "name": "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME", - "value": "batch-account-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID", - "value": "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" - }, - { - "name": "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID", - "value": "workload-user-assigned-identity-client-id" - } - ]""" - assert expected_settings == activity.type_properties["value"].value - assert expected_settings == state.get_variable_by_name("CommonEnvironmentSettings").value + env_settings = activity.type_properties["value"].get_json_value() + assert env_settings[0]["name"] == "WORKLOAD_APP_PACKAGE" + assert env_settings[0]["value"] == "workload" + assert env_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION" + assert env_settings[1]["value"] == "0.13.2" + assert env_settings[2]["name"] == "MANAGER_APP_PACKAGE" + assert env_settings[2]["value"] == "managerworkload" + assert env_settings[3]["name"] == "MANAGER_APP_PACKAGE_VERSION" + assert env_settings[3]["value"] == "0.13.2" + assert env_settings[4]["name"] == "BATCH_JOB_TIMEOUT" + assert env_settings[4]["value"] == "PT4H" + assert env_settings[5]["name"] == "WORKLOAD_AUTO_STORAGE_ACCOUNT_NAME" + assert env_settings[5]["value"] == "batch-account-name" + assert env_settings[6]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_RESOURCE_ID" + assert ( + env_settings[6]["value"] + == "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" + ) + assert env_settings[7]["name"] == "WORKLOAD_USER_ASSIGNED_IDENTITY_CLIENT_ID" + assert env_settings[7]["value"] == "workload-user-assigned-identity-client-id" def test_create_job_storage_container(test_framework: TestFramework, pipeline: Pipeline) -> None: @@ -304,71 +288,52 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) - assert "/jobs?api-version=2022-10-01.16.0" == activity.type_properties["relativeUrl"] assert "POST" == activity.type_properties["method"] - expected_body = """{ - "id": "8b6b545b-c583-4a06-adf7-19ff41370aba", - "priority": 100, - "constraints": { - "maxWallClockTime":"PT4H", - "maxTaskRetryCount": 0 - }, - "jobManagerTask": { - "id": "Manager", - "displayName": "Manager", - "authenticationTokenSettings": { - "access": [ - "job" - ] - }, - "commandLine": "/bin/bash -c \\"python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_managerworkload_0_13_2/managerworkload.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz && python3 -m test-application-name job --parameter1 dummy --parameter2 another-dummy\\"", - "applicationPackageReferences": [ - { - "applicationId": "batchmanager", - "version": "2.0.0" - }, - { - "applicationId": "test-application-name", - "version": "1.5.0" - } - ], - "outputFiles": [ - { - "destination": { - "container": { - "containerUrl": "/job-8b6b545b-c583-4a06-adf7-19ff41370aba", - "identityReference": { - "resourceId": "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" - }, - "path": "Manager/$TaskLog" - } - }, - "filePattern": "../*.txt", - "uploadOptions": { - "uploadCondition": "taskcompletion" - } - } - ], - "environmentSettings": [], - "requiredSlots": 1, - "killJobOnCompletion": false, - "userIdentity": { - "username": null, - "autoUser": { - "scope": "pool", - "elevationLevel": "nonadmin" - } - }, - "runExclusive": true, - "allowLowPriorityNode": true - }, - "poolInfo": { - "poolId": "test-application-batch-pool-id" - }, - "onAllTasksComplete": "terminatejob", - "onTaskFailure": "noaction", - "usesTaskDependencies": true, - "commonEnvironmentSettings": [{"name": "COMMON_ENV_SETTING", "value": "dummy"}, {"name": "STORAGE_ACCOUNT_NAME", "value": "teststorage"}]}""" - - assert expected_body == activity.type_properties["body"].value + body = activity.type_properties["body"].get_json_value() + assert body["id"] == "8b6b545b-c583-4a06-adf7-19ff41370aba" + assert body["priority"] == 100 + assert body["constraints"]["maxWallClockTime"] == "PT4H" + assert body["constraints"]["maxTaskRetryCount"] == 0 + job_manager_task = body["jobManagerTask"] + assert job_manager_task["id"] == "Manager" + assert job_manager_task["displayName"] == "Manager" + assert job_manager_task["authenticationTokenSettings"]["access"] == ["job"] + assert ( + job_manager_task["commandLine"] + == '/bin/bash -c "python3 -m ensurepip --upgrade && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_managerworkload_0_13_2/managerworkload.tar.gz && python3 -m pip install --user $AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz && python3 -m test-application-name job --parameter1 dummy --parameter2 another-dummy"' + ) + application_package_references = job_manager_task["applicationPackageReferences"] + assert application_package_references[0]["applicationId"] == "batchmanager" + assert application_package_references[0]["version"] == "2.0.0" + assert application_package_references[1]["applicationId"] == "test-application-name" + assert application_package_references[1]["version"] == "1.5.0" + assert ( + job_manager_task["outputFiles"][0]["destination"]["container"]["containerUrl"] + == "/job-8b6b545b-c583-4a06-adf7-19ff41370aba" + ) + assert ( + job_manager_task["outputFiles"][0]["destination"]["container"]["identityReference"]["resourceId"] + == "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name" + ) + assert job_manager_task["outputFiles"][0]["destination"]["container"]["path"] == "Manager/$TaskLog" + assert job_manager_task["outputFiles"][0]["filePattern"] == "../*.txt" + assert job_manager_task["outputFiles"][0]["uploadOptions"]["uploadCondition"] == "taskcompletion" + assert job_manager_task["environmentSettings"] == [] + assert job_manager_task["requiredSlots"] == 1 + assert job_manager_task["killJobOnCompletion"] is False + assert job_manager_task["userIdentity"]["username"] is None + assert job_manager_task["userIdentity"]["autoUser"]["scope"] == "pool" + assert job_manager_task["userIdentity"]["autoUser"]["elevationLevel"] == "nonadmin" + assert job_manager_task["runExclusive"] is True + assert job_manager_task["allowLowPriorityNode"] is True + assert body["poolInfo"]["poolId"] == "test-application-batch-pool-id" + assert body["onAllTasksComplete"] == "terminatejob" + assert body["onTaskFailure"] == "noaction" + assert body["usesTaskDependencies"] is True + common_environment_settings = body["commonEnvironmentSettings"] + assert common_environment_settings[0]["name"] == "COMMON_ENV_SETTING" + assert common_environment_settings[0]["value"] == "dummy" + assert common_environment_settings[1]["name"] == "STORAGE_ACCOUNT_NAME" + assert common_environment_settings[1]["value"] == "teststorage" def test_monitor_job(test_framework: TestFramework, pipeline: Pipeline) -> None: diff --git a/src/data_factory_testing_framework/models/data_factory_element.py b/src/data_factory_testing_framework/models/data_factory_element.py index f4170144..9857e355 100644 --- a/src/data_factory_testing_framework/models/data_factory_element.py +++ b/src/data_factory_testing_framework/models/data_factory_element.py @@ -1,4 +1,5 @@ -from typing import Generic, TypeVar, Union +import json +from typing import Any, Generic, TypeVar, Union from data_factory_testing_framework.functions.expression_evaluator import ExpressionEvaluator from data_factory_testing_framework.state import RunState @@ -24,3 +25,10 @@ def evaluate(self, state: RunState) -> Union[str, int, bool, float]: evaluator = ExpressionEvaluator() self.value = evaluator.evaluate(self.expression, state) return self.value + + def get_json_value(self) -> Any: # noqa: ANN401 + """Loads the value as a json object.""" + if self.value: + return json.loads(self.value) + + return None