Skip to content

Commit

Permalink
refactor: package and DataFactoryElement interface
Browse files Browse the repository at this point in the history
  • Loading branch information
arjendev committed Feb 7, 2024
1 parent 4209b55 commit d3fc9ab
Show file tree
Hide file tree
Showing 28 changed files with 144 additions and 106 deletions.
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ The samples seen below are the _only_ code that you need to write! The framework
activity.evaluate(state)

# Assert
assert "https://example.com/jobs" == activity.type_properties["url"].value
assert "POST" == activity.type_properties["method"].value
assert "https://example.com/jobs" == activity.type_properties["url"].result
assert "POST" == activity.type_properties["method"].result
body = activity.type_properties["body"].get_json_value()
assert "123" == body["JobId"]
assert "Job-123" == body["JobName"]
Expand All @@ -129,19 +129,19 @@ The samples seen below are the _only_ code that you need to write! The framework
assert set_variable_activity is not None
assert "Set JobName" == set_variable_activity.name
assert "JobName" == activity.type_properties["variableName"]
assert "Job-123" == activity.type_properties["value"].value
assert "Job-123" == activity.type_properties["value"].result

get_version_activity = next(activities)
assert get_version_activity is not None
assert "Get version" == get_version_activity.name
assert "https://example.com/version" == get_version_activity.type_properties["url"].value
assert "https://example.com/version" == get_version_activity.type_properties["url"].result
assert "GET" == get_version_activity.type_properties["method"]
get_version_activity.set_result(DependencyCondition.Succeeded,{"Version": "version1"})

create_batch_activity = next(activities)
assert create_batch_activity is not None
assert "Trigger Azure Batch Job" == create_batch_activity.name
assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].value
assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].result
assert "POST" == create_batch_activity.type_properties["method"]
body = create_batch_activity.type_properties["body"].get_json_value()
assert "123" == body["JobId"]
Expand Down
10 changes: 5 additions & 5 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ The samples seen below is the *only* code that you need to write! The framework
activity.evaluate(state)

# Assert
assert "https://example.com/jobs" == activity.type_properties["url"].value
assert "POST" == activity.type_properties["method"].value
assert "https://example.com/jobs" == activity.type_properties["url"].result
assert "POST" == activity.type_properties["method"].result
body = activity.type_properties["body"].get_json_value()
assert "123" == body["JobId"]
assert "Job-123" == body["JobName"]
Expand All @@ -82,19 +82,19 @@ The samples seen below is the *only* code that you need to write! The framework
assert set_variable_activity is not None
assert "Set JobName" == set_variable_activity.name
assert "JobName" == activity.type_properties["variableName"]
assert "Job-123" == activity.type_properties["value"].value
assert "Job-123" == activity.type_properties["value"].result

get_version_activity = next(activities)
assert get_version_activity is not None
assert "Get version" == get_version_activity.name
assert "https://example.com/version" == get_version_activity.type_properties["url"].value
assert "https://example.com/version" == get_version_activity.type_properties["url"].result
assert "GET" == get_version_activity.type_properties["method"]
get_version_activity.set_result(DependencyCondition.Succeeded,{"Version": "version1"})

create_batch_activity = next(activities)
assert create_batch_activity is not None
assert "Trigger Azure Batch Job" == create_batch_activity.name
assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].value
assert "https://example.com/jobs" == create_batch_activity.type_properties["url"].result
assert "POST" == create_batch_activity.type_properties["method"]
body = create_batch_activity.type_properties["body"].get_json_value()
assert "123" == body["JobId"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,28 +61,28 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
assert activity.name == "Set UserAssignedIdentityReference"
assert activity.type_properties["variableName"] == "UserAssignedIdentityReference"
assert (
activity.type_properties["value"].value
activity.type_properties["value"].result
== "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.ManagedIdentity/userAssignedIdentities/test-application-identity-name" # noqa: E501
)

activity = next(activities)
assert activity.name == "Set ManagerApplicationPackagePath"
assert activity.type_properties["variableName"] == "ManagerApplicationPackagePath"
assert activity.type_properties["value"].value == "$AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz"
assert activity.type_properties["value"].result == "$AZ_BATCH_APP_PACKAGE_batchmanager_2_0_0/batchmanager.tar.gz"

activity = next(activities)
assert activity.name == "Set WorkloadApplicationPackagePath"
assert activity.type_properties["variableName"] == "WorkloadApplicationPackagePath"
assert (
activity.type_properties["value"].value
activity.type_properties["value"].result
== "$AZ_BATCH_APP_PACKAGE_test-application_1_5_0/test-application.tar.gz"
)

activity = next(activities)
assert activity.name == "Set CommonEnvironmentSettings"
assert activity.type_properties["variableName"] == "CommonEnvironmentSettings"

common_environment_settings = activity.type_properties["value"].value
common_environment_settings = activity.type_properties["value"].result
assert len(common_environment_settings) == 8
assert common_environment_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
assert common_environment_settings[0]["value"] == "test-application"
Expand Down Expand Up @@ -110,29 +110,29 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
activity = next(activities)
assert activity.name == "Set JobContainerName"
assert activity.type_properties["variableName"] == "JobContainerName"
assert activity.type_properties["value"].value == "job-802100a5-ec79-4a52-be62-8d6109f3ff9a"
assert activity.type_properties["value"].result == "job-802100a5-ec79-4a52-be62-8d6109f3ff9a"

activity = next(activities)
assert activity.name == "Set Job Container URL"
assert activity.type_properties["variableName"] == "JobContainerURL"
assert (
activity.type_properties["value"].value
activity.type_properties["value"].result
== "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a"
)

activity = next(activities)
assert activity.name == "Create Job Storage Container"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a?restype=container"
)
assert activity.type_properties["method"] == "PUT"
assert activity.type_properties["body"].value == "{}"
assert activity.type_properties["body"].result == "{}"

activity = next(activities)
assert activity.name == "Start Job"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://batch-account-name.westeurope.batch.azure.com/jobs?api-version=2022-10-01.16.0"
)
assert activity.type_properties["method"] == "POST"
Expand Down Expand Up @@ -192,29 +192,29 @@ def test_batch_job_pipeline(request: pytest.FixtureRequest) -> None:
assert activity.name == "Monitor Batch Job"
assert activity.type_properties["pipeline"]["referenceName"] == "monitor_batch_job"
assert len(activity.type_properties["parameters"]) == 1
assert activity.type_properties["parameters"]["JobId"].value == "802100a5-ec79-4a52-be62-8d6109f3ff9a"
assert activity.type_properties["parameters"]["JobId"].result == "802100a5-ec79-4a52-be62-8d6109f3ff9a"

activity = next(activities)
assert activity.name == "Copy Output Files"
assert activity.type_properties["pipeline"]["referenceName"] == "copy_output_files"
assert len(activity.type_properties["parameters"]) == 5
assert (
activity.type_properties["parameters"]["JobContainerName"].value == "job-802100a5-ec79-4a52-be62-8d6109f3ff9a"
activity.type_properties["parameters"]["JobContainerName"].result == "job-802100a5-ec79-4a52-be62-8d6109f3ff9a"
)
assert activity.type_properties["parameters"]["TaskOutputFolderPrefix"].value == "TASKOUTPUT_"
assert activity.type_properties["parameters"]["TaskOutputFolderPrefix"].result == "TASKOUTPUT_"
assert (
activity.type_properties["parameters"]["OutputStorageAccountName"].value
activity.type_properties["parameters"]["OutputStorageAccountName"].result
== "test-application-output-storage-account-name"
)
assert (
activity.type_properties["parameters"]["OutputContainerName"].value == "test-application-output-container-name"
activity.type_properties["parameters"]["OutputContainerName"].result == "test-application-output-container-name"
)
assert activity.type_properties["parameters"]["OutputFolderName"].value == "TEMP"
assert activity.type_properties["parameters"]["OutputFolderName"].result == "TEMP"

activity = next(activities)
assert activity.name == "Delete Job Storage Container"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://batch-account-name.blob.core.windows.net/job-802100a5-ec79-4a52-be62-8d6109f3ff9a?restype=container"
)
assert activity.type_properties["method"] == "DELETE"
Expand Down
32 changes: 16 additions & 16 deletions examples/data_factory/batch_job/test_data_factory_batchjob_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def test_set_job_container_url(test_framework: TestFramework, pipeline: Pipeline
# Assert
updated_variable = state.get_variable_by_name("JobContainerURL")
expected_url = "https://batch-account-name.blob.core.windows.net/job-8b6b545b-c583-4a06-adf7-19ff41370aba"
assert activity.type_properties["value"].value == expected_url
assert activity.type_properties["value"].result == expected_url
assert updated_variable.value == expected_url


Expand Down Expand Up @@ -70,7 +70,7 @@ def test_set_user_assigned_identity_reference(test_framework: TestFramework, pip
# Assert
updated_variable = state.get_variable_by_name("UserAssignedIdentityReference")
expected_reference = "/subscriptions/batch-account-subscription/resourcegroups/batch-account-resource-group/providers/Microsoft.ManagedIdentity/userAssignedIdentities/workload-user-assigned-identity-name"
assert activity.type_properties["value"].value == expected_reference
assert activity.type_properties["value"].result == expected_reference
assert updated_variable.value == expected_reference


Expand All @@ -93,7 +93,7 @@ def test_set_manager_application_package_path(test_framework: TestFramework, pip
# Assert
updated_variable = state.get_variable_by_name("ManagerApplicationPackagePath")
expected_path = "$AZ_BATCH_APP_PACKAGE_managerworkload_0_13_2/managerworkload.tar.gz"
assert activity.type_properties["value"].value == expected_path
assert activity.type_properties["value"].result == expected_path
assert updated_variable.value == expected_path


Expand All @@ -116,7 +116,7 @@ def test_set_workload_application_package_path(test_framework: TestFramework, pi
# Assert
updated_variable = state.get_variable_by_name("WorkloadApplicationPackagePath")
expected_path = "$AZ_BATCH_APP_PACKAGE_workload_0_13_2/workload.tar.gz"
assert activity.type_properties["value"].value == expected_path
assert activity.type_properties["value"].result == expected_path
assert updated_variable.value == expected_path


Expand Down Expand Up @@ -155,7 +155,7 @@ def test_set_common_environment_settings(test_framework: TestFramework, pipeline
activity.evaluate(state)

# Assert
env_settings = activity.type_properties["value"].value
env_settings = activity.type_properties["value"].result
assert env_settings[0]["name"] == "WORKLOAD_APP_PACKAGE"
assert env_settings[0]["value"] == "workload"
assert env_settings[1]["name"] == "WORKLOAD_APP_PACKAGE_VERSION"
Expand Down Expand Up @@ -195,11 +195,11 @@ def test_create_job_storage_container(test_framework: TestFramework, pipeline: P
# Assert
assert activity.name == "Create Job Storage Container"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://batchstorage.blob.core.windows.net/job-8b6b545b-c583-4a06-adf7-19ff41370aba?restype=container"
)
assert activity.type_properties["method"] == "PUT"
assert activity.type_properties["body"].value == "{}"
assert activity.type_properties["body"].result == "{}"


def test_set_job_container_name(test_framework: TestFramework, pipeline: Pipeline) -> None:
Expand All @@ -217,7 +217,7 @@ def test_set_job_container_name(test_framework: TestFramework, pipeline: Pipelin

# Assert
job_container_name_variable = state.get_variable_by_name("JobContainerName")
assert activity.type_properties["value"].value == "job-8b6b545b-c583-4a06-adf7-19ff41370aba"
assert activity.type_properties["value"].result == "job-8b6b545b-c583-4a06-adf7-19ff41370aba"
assert job_container_name_variable.value == "job-8b6b545b-c583-4a06-adf7-19ff41370aba"


Expand Down Expand Up @@ -293,7 +293,7 @@ def test_start_job_pipeline(test_framework: TestFramework, pipeline: Pipeline) -
assert "Start Job" == activity.name
assert (
"https://batch-account-name.westeurope.batch.azure.com/jobs?api-version=2022-10-01.16.0"
== activity.type_properties["url"].value
== activity.type_properties["url"].result
)
assert "POST" == activity.type_properties["method"]

Expand Down Expand Up @@ -361,7 +361,7 @@ def test_monitor_job(test_framework: TestFramework, pipeline: Pipeline) -> None:
assert activity.name == "Monitor Batch Job"
assert activity.type_properties["pipeline"]["referenceName"] == "monitor_batch_job"
assert len(activity.type_properties["parameters"]) == 1
assert activity.type_properties["parameters"]["JobId"].value == "8b6b545b-c583-4a06-adf7-19ff41370aba"
assert activity.type_properties["parameters"]["JobId"].result == "8b6b545b-c583-4a06-adf7-19ff41370aba"


def test_copy_output_files(test_framework: TestFramework, pipeline: Pipeline) -> None:
Expand Down Expand Up @@ -391,14 +391,14 @@ def test_copy_output_files(test_framework: TestFramework, pipeline: Pipeline) ->
assert activity.type_properties["pipeline"]["referenceName"] == "copy_output_files"
assert len(activity.type_properties["parameters"]) == 5
assert (
activity.type_properties["parameters"]["JobContainerName"].value == "job-8b6b545b-c583-4a06-adf7-19ff41370aba"
activity.type_properties["parameters"]["JobContainerName"].result == "job-8b6b545b-c583-4a06-adf7-19ff41370aba"
)
assert activity.type_properties["parameters"]["TaskOutputFolderPrefix"].value == "TASKOUTPUT_"
assert activity.type_properties["parameters"]["OutputStorageAccountName"].value == "teststorage"
assert activity.type_properties["parameters"]["TaskOutputFolderPrefix"].result == "TASKOUTPUT_"
assert activity.type_properties["parameters"]["OutputStorageAccountName"].result == "teststorage"
assert (
activity.type_properties["parameters"]["OutputContainerName"].value == "test-application-output-container-name"
activity.type_properties["parameters"]["OutputContainerName"].result == "test-application-output-container-name"
)
assert activity.type_properties["parameters"]["OutputFolderName"].value == "output"
assert activity.type_properties["parameters"]["OutputFolderName"].result == "output"


def test_delete_job_storage_container(test_framework: TestFramework, pipeline: Pipeline) -> None:
Expand All @@ -419,7 +419,7 @@ def test_delete_job_storage_container(test_framework: TestFramework, pipeline: P
# Assert
assert activity.name == "Delete Job Storage Container"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://batchstorage.blob.core.windows.net/job-8b6b545b-c583-4a06-adf7-19ff41370aba?restype=container"
)
assert activity.type_properties["method"] == "DELETE"
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def test_copy_blobs_pipeline(request: pytest.FixtureRequest) -> None:
list_folder_activity = next(activities)
assert list_folder_activity.name == "List Folders"
assert (
list_folder_activity.type_properties["url"].value
list_folder_activity.type_properties["url"].result
== "https://sourcestorageaccount.blob.core.windows.net/sourcecontainer?restype=container&comp=list&prefix=sourcefolder&delimiter=$SourceBlobDelimiter"
)
assert list_folder_activity.type_properties["method"] == "GET"
Expand Down Expand Up @@ -60,15 +60,15 @@ def test_copy_blobs_pipeline(request: pytest.FixtureRequest) -> None:
assert copy_activity.name == "Copy files to Destination"
assert copy_activity.type == "Copy"
assert (
copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].value
copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result
== "testfolder_1/$SourceBlobDelimiter"
)

copy_activity = next(activities)
assert copy_activity.name == "Copy files to Destination"
assert copy_activity.type == "Copy"
assert (
copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].value
copy_activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result
== "testfolder_2/$SourceBlobDelimiter"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def test_list_blobs(pipeline: Pipeline) -> None:
# Assert
assert activity.name == "List Folders"
assert (
activity.type_properties["url"].value
activity.type_properties["url"].result
== "https://sourcestorage.blob.core.windows.net/container-8b6b545b-c583-4a06-adf7-19ff41370aba?restype=container&comp=list&prefix=testfolder&delimiter=$SourceBlobDelimiter"
)
assert activity.type_properties["method"] == "GET"
Expand Down Expand Up @@ -93,7 +93,7 @@ def test_for_each(pipeline: Pipeline) -> None:

# Assert
assert activity.name == "For Each SourceFolder"
assert activity.type_properties["items"].value == [
assert activity.type_properties["items"].result == [
"testfolder_1/$SourceBlobDelimiter",
"testfolder_2/$SourceBlobDelimiter",
]
Expand Down Expand Up @@ -135,4 +135,4 @@ def test_copy_blobs_activity(pipeline: Pipeline, wildcardfolderpath: str) -> Non

# Assert
assert activity.name == "Copy files to Destination"
assert activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].value == wildcardfolderpath
assert activity.type_properties["source"]["storeSettings"]["wildcardFolderPath"].result == wildcardfolderpath
Loading

0 comments on commit d3fc9ab

Please sign in to comment.