From ecb868c05f792a3bea5057f3f557604649590e4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:06:11 +0000 Subject: [PATCH 001/100] Bump googleapis-common-protos in /community/front-end/ofe Bumps [googleapis-common-protos](https://github.com/googleapis/python-api-common-protos) from 1.54.0 to 1.58.0. - [Release notes](https://github.com/googleapis/python-api-common-protos/releases) - [Changelog](https://github.com/googleapis/python-api-common-protos/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/python-api-common-protos/compare/v1.54.0...v1.58.0) --- updated-dependencies: - dependency-name: googleapis-common-protos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 6427f690d0..eb3e45b310 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -26,7 +26,7 @@ google-cloud-pubsub==2.9.0 google-cloud-storage==2.1.0 google-crc32c==1.3.0 google-resumable-media==2.2.1 -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.58.0 grafana_api==1.0.3 grpc-google-iam-v1==0.12.3 grpcio==1.43.0 From bf439120d5f569bcf3c3adf2557d34469efc4020 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:06:19 +0000 Subject: [PATCH 002/100] Bump httplib2 from 0.20.4 to 0.21.0 in /community/front-end/ofe Bumps [httplib2](https://github.com/httplib2/httplib2) from 0.20.4 to 0.21.0. - [Release notes](https://github.com/httplib2/httplib2/releases) - [Changelog](https://github.com/httplib2/httplib2/blob/master/CHANGELOG) - [Commits](https://github.com/httplib2/httplib2/compare/v0.20.4...v0.21.0) --- updated-dependencies: - dependency-name: httplib2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 6427f690d0..8846d80f62 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -32,7 +32,7 @@ grpc-google-iam-v1==0.12.3 grpcio==1.43.0 grpcio-status==1.43.0 h11==0.13.0 -httplib2==0.20.4 +httplib2==0.21.0 idna==3.3 isort==5.12.0 lazy-object-proxy==1.7.1 From 969059938f664ce7215e4ec0e89c864fc0b75bf3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:06:30 +0000 Subject: [PATCH 003/100] Bump wrapt from 1.13.3 to 1.15.0 in /community/front-end/ofe Bumps [wrapt](https://github.com/GrahamDumpleton/wrapt) from 1.13.3 to 1.15.0. - [Release notes](https://github.com/GrahamDumpleton/wrapt/releases) - [Changelog](https://github.com/GrahamDumpleton/wrapt/blob/develop/docs/changes.rst) - [Commits](https://github.com/GrahamDumpleton/wrapt/compare/1.13.3...1.15.0) --- updated-dependencies: - dependency-name: wrapt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 6427f690d0..2a5dd123dc 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -67,6 +67,6 @@ typing_extensions==4.1.1 uritemplate==4.1.1 urllib3==1.26.8 uvicorn==0.17.4 -wrapt==1.13.3 +wrapt==1.15.0 xmltodict==0.12.0 yq==2.13.0 From 1229a101902dd107a5d5d4173d8930c6cf93ecd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:06:41 +0000 Subject: [PATCH 004/100] Bump django-extensions from 3.1.5 to 3.2.1 in /community/front-end/ofe Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.1.5 to 3.2.1. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.1.5...3.2.1) --- updated-dependencies: - dependency-name: django-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 6427f690d0..ead0c4db64 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -12,7 +12,7 @@ defusedxml==0.7.1 dill==0.3.6 Django==3.2.18 django-allauth==0.48.0 -django-extensions==3.1.5 +django-extensions==3.2.1 # Need version 0.11.0 to be released with fixes for Django 3.2 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 djangorestframework==3.13.1 From af35483396ef6cb09fb4e0b9dd89aaa556704871 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 13 Mar 2023 15:21:05 -0500 Subject: [PATCH 005/100] HTCondor MIG enhancement Migrate settings for each MIG autoscaler to a SystemD configuration file --- .../files/htcondor_configure_autoscaler.yml | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml index 1b2152deaf..058f8abf48 100644 --- a/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ b/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -45,7 +45,26 @@ [Service] User=condor Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p {{ project_id }} --r {{ region }} --z {{ zone }} --mz --g %i --c {{ max_size }} + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE + notify: + - Reload SystemD + - name: Create SystemD override directory for autoscaler configuration + ansible.builtin.file: + path: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d" + state: directory + owner: root + group: root + mode: 0755 + - name: Create autoscaler configuration + ansible.builtin.copy: + dest: "{{ systemd_override_path }}/htcondor-autoscaler@{{ mig_id }}.service.d/miglimit.conf" + mode: 0644 + content: | + [Service] + Environment=PROJECT_ID={{ project_id }} + Environment=REGION={{ region }} + Environment=ZONE={{ zone }} + Environment=MAX_SIZE={{ max_size }} notify: - Reload SystemD - name: Create SystemD timer for HTCondor autoscaler From 39e1679bd8795a07c717dd3d06e43f160e179e11 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 13 Mar 2023 15:21:05 -0500 Subject: [PATCH 006/100] Configure HTCondor Negotiator Will now update Classad immediately upon negotiation cycle --- .../scheduler/htcondor-configure/files/htcondor_configure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/community/modules/scheduler/htcondor-configure/files/htcondor_configure.yml b/community/modules/scheduler/htcondor-configure/files/htcondor_configure.yml index 25b4140e2f..6518c31d8e 100644 --- a/community/modules/scheduler/htcondor-configure/files/htcondor_configure.yml +++ b/community/modules/scheduler/htcondor-configure/files/htcondor_configure.yml @@ -94,6 +94,7 @@ COLLECTOR_UPDATE_INTERVAL=30 NEGOTIATOR_UPDATE_INTERVAL=30 NEGOTIATOR_DEPTH_FIRST=True + NEGOTIATOR_UPDATE_AFTER_CYCLE=True notify: - Reload HTCondor - name: Create Central Manager HA configuration file From cdb22bd01b116f8c6b27d57df17cf3e5f27358cc Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 13 Mar 2023 15:21:05 -0500 Subject: [PATCH 007/100] HTCondor MIG enhancement Add support for minimum # of idle VMs --- .../compute/htcondor-execute-point/README.md | 3 +- .../files/htcondor_configure_autoscaler.yml | 3 +- .../compute/htcondor-execute-point/main.tf | 9 +- .../htcondor-execute-point/variables.tf | 8 +- .../htcondor-install/files/autoscaler.py | 215 +++++++++++------- 5 files changed, 146 insertions(+), 92 deletions(-) diff --git a/community/modules/compute/htcondor-execute-point/README.md b/community/modules/compute/htcondor-execute-point/README.md index 55713759e2..8ccf4f61a3 100644 --- a/community/modules/compute/htcondor-execute-point/README.md +++ b/community/modules/compute/htcondor-execute-point/README.md @@ -184,8 +184,9 @@ No resources. | [image](#input\_image) | HTCondor execute point VM image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | | [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | | [machine\_type](#input\_machine\_type) | Machine type to use for HTCondor execute points | `string` | `"n2-standard-4"` | no | -| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool; set to constrain cost run-away. | `number` | `100` | no | +| [max\_size](#input\_max\_size) | Maximum size of the HTCondor execute point pool. | `number` | `100` | no | | [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | +| [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | | [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | | [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | diff --git a/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml b/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml index 058f8abf48..c8dfd9bb9e 100644 --- a/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml +++ b/community/modules/compute/htcondor-execute-point/files/htcondor_configure_autoscaler.yml @@ -45,7 +45,7 @@ [Service] User=condor Type=oneshot - ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE + ExecStart={{ python }} {{ autoscaler }} --p $PROJECT_ID --r $REGION --z $ZONE --mz --g %i --c $MAX_SIZE --i $MIN_IDLE notify: - Reload SystemD - name: Create SystemD override directory for autoscaler configuration @@ -65,6 +65,7 @@ Environment=REGION={{ region }} Environment=ZONE={{ zone }} Environment=MAX_SIZE={{ max_size }} + Environment=MIN_IDLE={{ min_idle }} notify: - Reload SystemD - name: Create SystemD timer for HTCondor autoscaler diff --git a/community/modules/compute/htcondor-execute-point/main.tf b/community/modules/compute/htcondor-execute-point/main.tf index 7926ed6dea..db99f23065 100644 --- a/community/modules/compute/htcondor-execute-point/main.tf +++ b/community/modules/compute/htcondor-execute-point/main.tf @@ -30,7 +30,14 @@ locals { "type" = "ansible-local" "content" = file("${path.module}/files/htcondor_configure_autoscaler.yml") "destination" = "htcondor_configure_autoscaler_${module.mig.instance_group_manager.name}.yml" - "args" = "-e project_id=${var.project_id} -e region=${var.region} -e zone=${var.zone} -e mig_id=${module.mig.instance_group_manager.name} -e max_size=${var.max_size}" + "args" = join(" ", [ + "-e project_id=${var.project_id}", + "-e region=${var.region}", + "-e zone=${var.zone}", + "-e mig_id=${module.mig.instance_group_manager.name}", + "-e max_size=${var.max_size}", + "-e min_idle=${var.min_idle}", + ]) } hostnames = var.spot ? "${var.deployment_name}-spot-xp" : "${var.deployment_name}-xp" diff --git a/community/modules/compute/htcondor-execute-point/variables.tf b/community/modules/compute/htcondor-execute-point/variables.tf index c08c65bd88..37a7f476bf 100644 --- a/community/modules/compute/htcondor-execute-point/variables.tf +++ b/community/modules/compute/htcondor-execute-point/variables.tf @@ -108,11 +108,17 @@ variable "target_size" { } variable "max_size" { - description = "Maximum size of the HTCondor execute point pool; set to constrain cost run-away." + description = "Maximum size of the HTCondor execute point pool." type = number default = 100 } +variable "min_idle" { + description = "Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly." + type = number + default = 0 +} + variable "metadata" { description = "Metadata to add to HTCondor execute points" type = map(string) diff --git a/community/modules/scripts/htcondor-install/files/autoscaler.py b/community/modules/scripts/htcondor-install/files/autoscaler.py index cb4aacd831..e4605d6099 100644 --- a/community/modules/scripts/htcondor-install/files/autoscaler.py +++ b/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # -*- coding: utf-8 -*- # Copyright 2018 Google Inc. All Rights Reserved. @@ -20,6 +20,7 @@ from absl import app from absl import flags +from collections import OrderedDict from pprint import pprint from googleapiclient import discovery from oauth2client.client import GoogleCredentials @@ -31,7 +32,7 @@ import htcondor import classad -parser = argparse.ArgumentParser() +parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--p", required=True, help="Project id", type=str) parser.add_argument( "--z", @@ -54,6 +55,12 @@ parser.add_argument( "--g", required=True, help="Name of the managed instance group", type=str ) +parser.add_argument( + "--i", + default=0, + help="Minimum number of idle compute instances", + type=int +) parser.add_argument( "--c", required=True, help="Maximum number of compute instances", type=int ) @@ -87,26 +94,19 @@ def __init__(self, multizone=False): else: self.instanceGroupManagers = self.service.instanceGroupManagers() - # Remove specified instance from MIG and decrease MIG size - def deleteFromMig(self, instance, zone): - instanceUrl = "https://www.googleapis.com/compute/v1/projects/" + self.project - instanceUrl += "/zones/" + zone - instanceUrl += "/instances/" + instance - - instances_to_delete = {"instances": [instanceUrl]} - + # Remove specified instances from MIG and decrease MIG size + def deleteFromMig(self, node_self_links): requestDelInstance = self.instanceGroupManagers.deleteInstances( project=self.project, **self.zoneargs, instanceGroupManager=self.instance_group_manager, - body=instances_to_delete, + body={ "instances": node_self_links }, ) # execute if not a dry-run if not self.dryrun: response = requestDelInstance.execute() if self.debug > 0: - print("Request to delete instance " + instance) pprint(response) return response return "Dry Run" @@ -185,6 +185,10 @@ def scale(self): # stewardship. A full list of Job ClassAd attributes can be found at # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html schedd = htcondor.Schedd() + # encourage the job queue to start a new negotiation cycle + # there are internal unconfigurable rate limits so not guaranteed + schedd.reschedule() + time.sleep(3) REQUEST_CPUS_ATTRIBUTE = "RequestCpus" REQUEST_GPUS_ATTRIBUTE = "RequestGpus" REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" @@ -206,49 +210,37 @@ def scale(self): spot_query = classad.ExprTree(f"RequireSpot == {self.is_spot}") # For purpose of scaling a Managed Instance Group, count only jobs that - # are running or idle ("potentially runnable"). Filter JobStatus by: + # are idle and likely participated in a negotiation cycle (there does + # not appear to be a single classad attribute for this). # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html#JobStatus - running_job_query = classad.ExprTree("JobStatus == 2") - idle_job_query = classad.ExprTree("JobStatus == 1") - running_job_ads = schedd.query(constraint=running_job_query.and_(spot_query), - projection=job_attributes) + LAST_CYCLE_ATTRIBUTE = "LastNegotiationCycleTime0" + coll = htcondor.Collector() + negotiator_ad = coll.query(htcondor.AdTypes.Negotiator, projection=[LAST_CYCLE_ATTRIBUTE]) + if len(negotiator_ad) != 1: + print(f"There should be exactly 1 negotiator in the pool. There is {len(negotiator_ad)}") + exit() + last_negotiation_cycle_time = negotiator_ad[0].get(LAST_CYCLE_ATTRIBUTE) + if not last_negotiation_cycle_time: + print(f"The negotiator has not yet started a match cycle. Exiting auto-scaling.") + exit() + + print(f"Last negotiation cycle occurred at: {last_negotiation_cycle_time}") + idle_job_query = classad.ExprTree(f"JobStatus == 1 && QDate < {last_negotiation_cycle_time}") idle_job_ads = schedd.query(constraint=idle_job_query.and_(spot_query), projection=job_attributes) total_idle_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in idle_job_ads) - total_running_request_cpus = sum(j[REQUEST_CPUS_ATTRIBUTE] for j in running_job_ads) - queue = total_idle_request_cpus + total_running_request_cpus - - print(f"Total CPUs requested by running jobs: {total_running_request_cpus}") print(f"Total CPUs requested by idle jobs: {total_idle_request_cpus}") if self.debug > 1: print("Information about the compute instance template") pprint(instanceTemplateInfo) - # Calculate number instances to satisfy current job queue CPU requests - if queue > 0: - self.size = int(math.ceil(float(queue) / float(self.cores_per_node))) - if self.debug > 0: - print( - "Calculating size of MIG: ⌈" - + str(queue) - + "/" - + str(self.cores_per_node) - + "⌉ = " - + str(self.size) - ) - else: - self.size = 0 - - # If compute instance limit is specified, can not start more instances then specified in the limit - if self.compute_instance_limit > 0 and self.size > self.compute_instance_limit: - self.size = self.compute_instance_limit - print( - "MIG target size will be limited by " + str(self.compute_instance_limit) - ) - - print("New MIG target size: " + str(self.size)) + # Calculate the minimum number of instances that, for fully packed + # execute points, could satisfy current job queue + min_hosts_for_idle_jobs = math.ceil(total_idle_request_cpus / self.cores_per_node) + if self.debug > 0: + print(f"Minimum hosts needed: {total_idle_request_cpus} / {self.cores_per_node} = {min_hosts_for_idle_jobs}") # Get current number of instances in the MIG requestGroupInfo = self.instanceGroupManagers.get( @@ -257,65 +249,110 @@ def scale(self): instanceGroupManager=self.instance_group_manager, ) responseGroupInfo = requestGroupInfo.execute() - currentTarget = int(responseGroupInfo["targetSize"]) - print("Current MIG target size: " + str(currentTarget)) + current_target = responseGroupInfo["targetSize"] + print(f"Current MIG target size: {current_target}") + + being_born_states = ["CREATING", "RECREATING", "VERIFYING"] + being_born_filters = [ f"currentAction = \"{state}\"" for state in being_born_states ] + being_born_combined_filter = ' OR '.join(being_born_filters) + reqCreatingInstances = self.instanceGroupManagers.listManagedInstances( + project=self.project, + **self.zoneargs, + instanceGroupManager=self.instance_group_manager, + filter=being_born_combined_filter, + orderBy="creationTimestamp desc" + ) + respCreatingInstances = reqCreatingInstances.execute() + + # Find VMs that are idle (no dynamic slots created from partitionable + # slots) in the MIG handled by this autoscaler + filter_idle_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots==0") + filter_claimed_vms = classad.ExprTree(f"PartitionableSlot && NumDynamicSlots>0") + filter_mig = classad.ExprTree(f"regexp(\".*/{self.instance_group_manager}$\", CloudCreatedBy)") + # A full list of Machine (StartD) ClassAd attributes can be found at + # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html + idle_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_idle_vms.and_(filter_mig), + projection=["Machine", "CloudZone"]) + + NODENAME_ATTRIBUTE = "UtsnameNodename" + claimed_node_ads = coll.query(htcondor.AdTypes.Startd, + constraint=filter_claimed_vms.and_(filter_mig), + projection=[NODENAME_ATTRIBUTE]) + claimed_nodes = [ ad[NODENAME_ATTRIBUTE] for ad in claimed_node_ads] + + # treat OrderedDict as a set by ignoring key values; this set will + # contain VMs we would consider deleting, in inverse order of + # their readiness to join pool (creating, unhealthy, healthy+idle) + idle_nodes = OrderedDict() + try: + creatingInstances = respCreatingInstances["managedInstances"] + except KeyError: + creatingInstances = [] + + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be running jobs + for instance in creatingInstances: + self_link = instance["instance"] + node_name = self_link.rsplit("/", 1)[-1] + if node_name not in claimed_nodes: + idle_nodes[self_link] = "creating" + + for ad in idle_node_ads: + node = ad["Machine"].split(".")[0] + zone = ad["CloudZone"] + self_link = "https://www.googleapis.com/compute/v1/projects/" + \ + self.project + "/zones/" + zone + "/instances/" + node + # there is potential for nodes in MIG health check "VERIFYING" state + # to have already joined the pool and be idle; delete them last + if self_link in idle_nodes: + idle_nodes.move_to_end(self_link) + idle_nodes[self_link] = "idle" + n_idle = len(idle_nodes) + + print(f"There are {n_idle} VMs being created or idle in the pool") + if self.debug > 1: + print("Listing idle nodes:") + pprint(idle_nodes) + + # always keep size tending toward the minimum idle VMs requested + new_target = current_target + self.compute_instance_min_idle - n_idle + min_hosts_for_idle_jobs + if new_target > self.compute_instance_limit: + self.size = self.compute_instance_limit + print(f"MIG target size will be limited by {self.compute_instance_limit}") + else: + self.size = new_target + + print(f"New MIG target size: {self.size}") if self.debug > 1: print("MIG Information:") print(responseGroupInfo) - if self.size == 0 and currentTarget == 0: - print( - "No jobs in the queue and no compute instances running. Nothing to do" - ) - exit() - - if self.size == currentTarget: - print( - "Running correct number of compute nodes to handle number of jobs in the queue" - ) + if self.size == current_target: + if current_target == 0: + print("Queue is empty") + print("Running correct number of VMs to handle queue") exit() - if self.size < currentTarget: + if self.size < current_target: print("Scaling down. Looking for nodes that can be shut down") - # Find VMs that are unused (no dynamic slots created from - # partitionable slots) and have been booted for at least 150 seconds - max_daemon_start_time = int(time.time()-150) - filter_idle_vms = classad.ExprTree("PartitionableSlot && NumDynamicSlots==0") - filter_uptime = classad.ExprTree(f"DaemonStartTime<{max_daemon_start_time}") - - # A full list of Machine (StartD) ClassAd attributes can be found at - # https://htcondor.readthedocs.io/en/latest/classad-attributes/machine-classad-attributes.html - coll = htcondor.Collector() - idle_node_ads = coll.query(htcondor.AdTypes.Startd, - constraint=filter_idle_vms.and_(filter_uptime), - projection=["Machine", "CloudZone"]) - if self.debug > 1: print("Compute node busy status:") - for node_ad in idle_node_ads: - print(node_ad["Machine"]) - - # Shut down nodes that are not busy - for node_ad in idle_node_ads: - try: - node = node_ad["Machine"].split(".")[0] - zone = node_ad["CloudZone"] - except KeyError: - print(f"Skipping ad: {node_ad}") - continue - - print("Will shut down: " + node + " ...") - respDel = self.deleteFromMig(node, zone) - if self.debug > 1: - print("Shut down request for compute node " + node) - pprint(respDel) + for node in idle_nodes: + print(node) + + # Shut down idle nodes up to our calculated limit + nodes_to_delete = list(idle_nodes.keys())[0:current_target-self.size] + for node in nodes_to_delete: + print(f"Attempting to delete: {node.rsplit('/',1)[-1]}") + respDel = self.deleteFromMig(nodes_to_delete) if self.debug > 1: print("Scaling down complete") - if self.size > currentTarget: + if self.size > current_target: print( "Scaling up. Need to increase number of instances to " + str(self.size) ) @@ -355,6 +392,8 @@ def main(): # Default number of running instances that the managed instance group should maintain at any given time. This number will go up and down based on the load (number of jobs in the queue) scaler.size = 0 + scaler.compute_instance_min_idle = args.i + # Dry run: : 0, run scaling; 1, only provide info. scaler.dryrun = args.d > 0 From d471e825af36d60934dfda0befa08928b44b5e13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Mar 2023 18:13:37 +0000 Subject: [PATCH 008/100] Bump grpcio-status from 1.43.0 to 1.51.3 in /community/front-end/ofe Bumps [grpcio-status](https://grpc.io) from 1.43.0 to 1.51.3. --- updated-dependencies: - dependency-name: grpcio-status dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 6eafa4dca7..434ba52d77 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -30,7 +30,7 @@ googleapis-common-protos==1.58.0 grafana_api==1.0.3 grpc-google-iam-v1==0.12.3 grpcio==1.43.0 -grpcio-status==1.43.0 +grpcio-status==1.51.3 h11==0.13.0 httplib2==0.21.0 idna==3.3 From 8f738def29263fb414d99cb911423b101c78cd22 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Tue, 14 Mar 2023 11:24:33 -0700 Subject: [PATCH 009/100] Remove ModuleToGroup from DeploymentConfig (#1022) * Remove `ModuleToGroup` from `DeploymentConfig`, as it duplicates state. * Remove passing `modToGrp` around; * Fix bug in `expandSimpleVariable` IGC-case. --- pkg/config/config.go | 48 +++++++++--------- pkg/config/config_test.go | 21 +++++--- pkg/config/expand.go | 86 ++++++++++++++------------------- pkg/config/expand_test.go | 94 ++++++++++++------------------------ pkg/config/validator_test.go | 7 ++- 5 files changed, 112 insertions(+), 144 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7b21e0332f..546816a321 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -298,9 +298,7 @@ func (mc *ModConnection) isEmpty() (isEmpty bool) { type DeploymentConfig struct { Config Blueprint // Indexed by Resource Group name and Module Source - ModulesInfo map[string]map[string]modulereader.ModuleInfo - // Maps module ID to group index - ModuleToGroup map[string]int + ModulesInfo map[string]map[string]modulereader.ModuleInfo expanded bool moduleConnections []ModConnection } @@ -471,38 +469,47 @@ func validateGroupName(name string, usedNames map[string]bool) { // checkModuleAndGroupNames checks and imports module and resource group IDs // and names respectively. -func checkModuleAndGroupNames( - depGroups []DeploymentGroup) (map[string]int, error) { - moduleToGroup := make(map[string]int) +func checkModuleAndGroupNames(depGroups []DeploymentGroup) error { + seen := map[string]struct{}{} groupNames := make(map[string]bool) for iGrp, grp := range depGroups { validateGroupName(grp.Name, groupNames) for _, mod := range grp.Modules { - // Verify no duplicate module names - if _, ok := moduleToGroup[mod.ID]; ok { - return moduleToGroup, fmt.Errorf( - "%s: %s used more than once", errorMessages["duplicateID"], mod.ID) + + if _, ok := seen[mod.ID]; ok { + return fmt.Errorf("%s: %s used more than once", errorMessages["duplicateID"], mod.ID) } - moduleToGroup[mod.ID] = iGrp + seen[mod.ID] = struct{}{} // Verify Module Kind matches group Kind if grp.Kind == "" { depGroups[iGrp].Kind = mod.Kind } else if grp.Kind != mod.Kind { - return moduleToGroup, fmt.Errorf( + return fmt.Errorf( "%s: deployment group %s, got: %s, wanted: %s", errorMessages["mixedModule"], grp.Name, grp.Kind, mod.Kind) } } } - return moduleToGroup, nil + return nil +} + +func modToGrp(groups []DeploymentGroup, modID string) (int, error) { + i := slices.IndexFunc(groups, func(g DeploymentGroup) bool { + return slices.ContainsFunc(g.Modules, func(m Module) bool { + return m.ID == modID + }) + }) + if i == -1 { + return -1, fmt.Errorf("module %s was not found", modID) + } + return i, nil } // checkUsedModuleNames verifies that any used modules have valid names and // are in the correct group -func checkUsedModuleNames( - depGroups []DeploymentGroup, idToGroup map[string]int) error { +func checkUsedModuleNames(depGroups []DeploymentGroup) error { for _, grp := range depGroups { for _, mod := range grp.Modules { for _, usedMod := range mod.Use { @@ -510,8 +517,8 @@ func checkUsedModuleNames( if err != nil { return err } - err = ref.validate(depGroups, idToGroup) - if err != nil { + + if err = ref.validate(depGroups); err != nil { return err } @@ -561,13 +568,10 @@ func (dc *DeploymentConfig) validateConfig() { if err != nil { log.Fatal(err) } - moduleToGroup, err := checkModuleAndGroupNames(dc.Config.DeploymentGroups) - if err != nil { + if err = checkModuleAndGroupNames(dc.Config.DeploymentGroups); err != nil { log.Fatal(err) } - dc.ModuleToGroup = moduleToGroup - if err = checkUsedModuleNames( - dc.Config.DeploymentGroups, dc.ModuleToGroup); err != nil { + if err = checkUsedModuleNames(dc.Config.DeploymentGroups); err != nil { log.Fatal(err) } if err = checkBackends(dc.Config); err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d85b6dfa4b..b3c9cbb51f 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -353,6 +353,20 @@ func (s *MySuite) TestExpandConfig(c *C) { dc.ExpandConfig() } +func (s *MySuite) TestCheckModuleAndGroupNames(c *C) { + { // Duplicate module name same group + g := DeploymentGroup{Name: "ice", Modules: []Module{{ID: "pony"}, {ID: "pony"}}} + err := checkModuleAndGroupNames([]DeploymentGroup{g}) + c.Check(err, ErrorMatches, "module IDs must be unique: pony used more than once") + } + { // Duplicate module name different groups + ice := DeploymentGroup{Name: "ice", Modules: []Module{{ID: "pony"}}} + fire := DeploymentGroup{Name: "fire", Modules: []Module{{ID: "pony"}}} + err := checkModuleAndGroupNames([]DeploymentGroup{ice, fire}) + c.Check(err, ErrorMatches, "module IDs must be unique: pony used more than once") + } +} + func (s *MySuite) TestIsEmpty(c *C) { // Use connection is not empty conn := ModConnection{ @@ -540,13 +554,6 @@ func (s *MySuite) TestHasKind(c *C) { } -func (s *MySuite) TestCheckModuleAndGroupNames(c *C) { - dc := getDeploymentConfigForTest() - checkModuleAndGroupNames(dc.Config.DeploymentGroups) - testModID := dc.Config.DeploymentGroups[0].Modules[0].ID - c.Assert(dc.ModuleToGroup[testModID], Equals, 0) -} - func (s *MySuite) TestDeploymentName(c *C) { dc := getDeploymentConfigForTest() var e *InputValueError diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 5a9cc042a8..88fad8cdc8 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -450,7 +450,7 @@ func applyGlobalVarsInGroup( func updateGlobalVarTypes(vars map[string]interface{}) error { for k, v := range vars { - val, err := updateVariableType(v, varContext{}, make(map[string]int)) + val, err := updateVariableType(v, varContext{}) if err != nil { return fmt.Errorf("error setting type for deployment variable %s: %v", k, err) } @@ -596,16 +596,15 @@ func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varRefere return ref, nil } -func (ref *modReference) validate(depGroups []DeploymentGroup, modToGrp map[string]int) error { +func (ref *modReference) validate(depGroups []DeploymentGroup) error { callingModuleGroupIndex := slices.IndexFunc(depGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) if callingModuleGroupIndex == -1 { return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.FromGroupID) } - targetModuleGroupIndex, ok := modToGrp[ref.ID] - if !ok { - return fmt.Errorf("%s: module %s was not found", - errorMessages["varNotFound"], ref.ID) + targetModuleGroupIndex, err := modToGrp(depGroups, ref.ID) + if err != nil { + return err } targetModuleGroupName := depGroups[targetModuleGroupIndex].Name @@ -645,7 +644,7 @@ func (ref *modReference) validate(depGroups []DeploymentGroup, modToGrp map[stri // ref.ExplicitInterGroup: intergroup references must explicitly identify the // target group ID and intragroup references cannot have an incorrect explicit // group ID -func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]interface{}, modToGrp map[string]int) error { +func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]interface{}) error { // simplest case to evaluate is a deployment variable's existence if ref.ToGroupID == "deployment" { if ref.ID == "vars" { @@ -658,10 +657,9 @@ func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]i return fmt.Errorf("%s: %s", errorMessages["invalidDeploymentRef"], ref.ID) } - targetModuleGroupIndex, ok := modToGrp[ref.ID] - if !ok { - return fmt.Errorf("%s: module %s was not found", - errorMessages["varNotFound"], ref.ID) + targetModuleGroupIndex, err := modToGrp(depGroups, ref.ID) + if err != nil { + return err } targetModuleGroup := depGroups[targetModuleGroupIndex] @@ -721,9 +719,7 @@ func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]i } // Needs DeploymentGroups, variable string, current group, -func expandSimpleVariable( - context varContext, - modToGrp map[string]int) (string, error) { +func expandSimpleVariable(context varContext) (string, error) { // Get variable contents re := regexp.MustCompile(simpleVariableExp) @@ -742,8 +738,7 @@ func expandSimpleVariable( return "", err } - err = varRef.validate(context.blueprint.DeploymentGroups, context.blueprint.Vars, modToGrp) - if err != nil { + if err := varRef.validate(context.blueprint.DeploymentGroups, context.blueprint.Vars); err != nil { return "", err } @@ -756,13 +751,21 @@ func expandSimpleVariable( // intragroup reference can make direct reference to module output expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.ID, varRef.Name) default: - // intergroup reference; being by finding the target module in blueprint - toGrpIdx := modToGrp[varRef.ToGroupID] - toModIdx := slices.IndexFunc(context.blueprint.DeploymentGroups[toGrpIdx].Modules, func(m Module) bool { return m.ID == varRef.ID }) + + // intergroup reference; begin by finding the target module in blueprint + toGrpIdx := slices.IndexFunc( + context.blueprint.DeploymentGroups, + func(g DeploymentGroup) bool { return g.Name == varRef.ToGroupID }) + + if toGrpIdx == -1 { + return "", fmt.Errorf("invalid group reference: %s", varRef.ToGroupID) + } + toGrp := context.blueprint.DeploymentGroups[toGrpIdx] + toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.ID }) if toModIdx == -1 { return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.ID) } - toMod := &context.blueprint.DeploymentGroups[toGrpIdx].Modules[toModIdx] + toMod := toGrp.Modules[toModIdx] // ensure that the target module outputs the value in the root module // state and not just internally within its deployment group @@ -777,9 +780,7 @@ func expandSimpleVariable( return expandedVariable, nil } -func expandVariable( - context varContext, - modToGrp map[string]int) (string, error) { +func expandVariable(context varContext) (string, error) { re := regexp.MustCompile(anyVariableExp) matchall := re.FindAllString(context.varString, -1) errHint := "" @@ -824,18 +825,15 @@ func hasVariable(str string) bool { return matched } -func handleVariable( - prim interface{}, - context varContext, - modToGrp map[string]int) (interface{}, error) { +func handleVariable(prim interface{}, context varContext) (interface{}, error) { switch val := prim.(type) { case string: context.varString = val if hasVariable(val) { if isSimpleVariable(val) { - return expandSimpleVariable(context, modToGrp) + return expandSimpleVariable(context) } - return expandVariable(context, modToGrp) + return expandVariable(context) } return val, nil default: @@ -843,18 +841,14 @@ func handleVariable( } } -func updateVariableType( - value interface{}, - context varContext, - modToGrp map[string]int) (interface{}, error) { +func updateVariableType(value interface{}, context varContext) (interface{}, error) { var err error switch typedValue := value.(type) { case []interface{}: interfaceSlice := value.([]interface{}) { for i := 0; i < len(interfaceSlice); i++ { - interfaceSlice[i], err = updateVariableType( - interfaceSlice[i], context, modToGrp) + interfaceSlice[i], err = updateVariableType(interfaceSlice[i], context) if err != nil { return interfaceSlice, err } @@ -864,7 +858,7 @@ func updateVariableType( case map[string]interface{}: retMap := map[string]interface{}{} for k, v := range typedValue { - retMap[k], err = updateVariableType(v, context, modToGrp) + retMap[k], err = updateVariableType(v, context) if err != nil { return retMap, err } @@ -873,23 +867,20 @@ func updateVariableType( case map[interface{}]interface{}: retMap := map[string]interface{}{} for k, v := range typedValue { - retMap[k.(string)], err = updateVariableType(v, context, modToGrp) + retMap[k.(string)], err = updateVariableType(v, context) if err != nil { return retMap, err } } return retMap, err default: - return handleVariable(value, context, modToGrp) + return handleVariable(value, context) } } -func updateVariables( - context varContext, - interfaceMap map[string]interface{}, - modToGrp map[string]int) error { +func updateVariables(context varContext, interfaceMap map[string]interface{}) error { for key, value := range interfaceMap { - updatedVal, err := updateVariableType(value, context, modToGrp) + updatedVal, err := updateVariableType(value, context) if err != nil { return err } @@ -902,7 +893,7 @@ func updateVariables( // expands all variables func (dc *DeploymentConfig) expandVariables() error { for _, validator := range dc.Config.Validators { - err := updateVariables(varContext{blueprint: dc.Config}, validator.Inputs, make(map[string]int)) + err := updateVariables(varContext{blueprint: dc.Config}, validator.Inputs) if err != nil { return err @@ -916,10 +907,7 @@ func (dc *DeploymentConfig) expandVariables() error { modIndex: iMod, blueprint: dc.Config, } - err := updateVariables( - context, - mod.Settings, - dc.ModuleToGroup) + err := updateVariables(context, mod.Settings) if err != nil { return err } @@ -927,7 +915,7 @@ func (dc *DeploymentConfig) expandVariables() error { // ensure that variable references to projects in required APIs are expanded for projectID, requiredAPIs := range mod.RequiredApis { if isDeploymentVariable(projectID) { - s, err := handleVariable(projectID, varContext{blueprint: dc.Config}, make(map[string]int)) + s, err := handleVariable(projectID, varContext{blueprint: dc.Config}) if err != nil { return err } diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index f357486a7d..57b7ce1d5d 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -280,50 +280,49 @@ func (s *MySuite) TestUpdateVariableType(c *C) { // empty testSlice := []interface{}{} ctx := varContext{} - modToGrp := make(map[string]int) - ret, err := updateVariableType(testSlice, ctx, modToGrp) + ret, err := updateVariableType(testSlice, ctx) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // single string testSlice = append(testSlice, "string") - ret, err = updateVariableType(testSlice, ctx, modToGrp) + ret, err = updateVariableType(testSlice, ctx) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // add list testSlice = append(testSlice, []interface{}{}) - ret, err = updateVariableType(testSlice, ctx, modToGrp) + ret, err = updateVariableType(testSlice, ctx) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // add map testSlice = append(testSlice, make(map[string]interface{})) - ret, err = updateVariableType(testSlice, ctx, modToGrp) + ret, err = updateVariableType(testSlice, ctx) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // map, success testMap := make(map[string]interface{}) - ret, err = updateVariableType(testMap, ctx, modToGrp) + ret, err = updateVariableType(testMap, ctx) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add string testMap["string"] = "string" - ret, err = updateVariableType(testMap, ctx, modToGrp) + ret, err = updateVariableType(testMap, ctx) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add map testMap["map"] = make(map[string]interface{}) - ret, err = updateVariableType(testMap, ctx, modToGrp) + ret, err = updateVariableType(testMap, ctx) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add slice testMap["slice"] = []interface{}{} - ret, err = updateVariableType(testMap, ctx, modToGrp) + ret, err = updateVariableType(testMap, ctx) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // string, success testString := "string" - ret, err = updateVariableType(testString, ctx, modToGrp) + ret, err = updateVariableType(testString, ctx) c.Assert(err, IsNil) c.Assert(testString, DeepEquals, ret) } @@ -530,8 +529,6 @@ func (s *MySuite) TestIdentifyModuleByReference(c *C) { } func (s *MySuite) TestValidateModuleReference(c *C) { - var err error - dg := []DeploymentGroup{ { Name: "zero", @@ -554,15 +551,6 @@ func (s *MySuite) TestValidateModuleReference(c *C) { }, } - // strictly speaking this unit test now also tests this function - // we should expand the blueprint used in config_test.go to include - // multiple deployment groups - testModToGrp, err := checkModuleAndGroupNames(dg) - c.Assert(err, IsNil) - c.Assert(testModToGrp[dg[0].Modules[0].ID], Equals, 0) - c.Assert(testModToGrp[dg[0].Modules[1].ID], Equals, 0) - c.Assert(testModToGrp[dg[1].Modules[0].ID], Equals, 1) - // An intragroup reference from group 0 to module B in 0 (good) ref0ToB0 := modReference{ ID: dg[0].Modules[1].ID, @@ -570,8 +558,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[0].Name, Explicit: false, } - err = ref0ToB0.validate(dg, testModToGrp) - c.Assert(err, IsNil) + c.Assert(ref0ToB0.validate(dg), IsNil) // An explicit intergroup reference from group 1 to module A in 0 (good) xRef1ToA0 := modReference{ @@ -580,8 +567,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[1].Name, Explicit: true, } - err = xRef1ToA0.validate(dg, testModToGrp) - c.Assert(err, IsNil) + c.Assert(xRef1ToA0.validate(dg), IsNil) // An implicit intergroup reference from group 1 to module A in 0 (bad due to implicit) iRef1ToA0 := modReference{ @@ -590,8 +576,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[1].Name, Explicit: false, } - err = iRef1ToA0.validate(dg, testModToGrp) - c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupImplicit"])) + c.Assert(iRef1ToA0.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupImplicit"])) // An explicit intergroup reference from group 0 to module 1 in 1 (bad due to group ordering) xRefA0To1 := modReference{ @@ -600,8 +585,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[0].Name, Explicit: true, } - err = xRefA0To1.validate(dg, testModToGrp) - c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupOrder"])) + c.Assert(xRefA0To1.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupOrder"])) // An explicit intergroup reference from group 0 to B0 with a bad Group ID badRef0ToB0 := modReference{ @@ -610,8 +594,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[0].Name, Explicit: true, } - err = badRef0ToB0.validate(dg, testModToGrp) - c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["referenceWrongGroup"])) + c.Assert(badRef0ToB0.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["referenceWrongGroup"])) // A target module that doesn't exist (bad) badTargetMod := modReference{ @@ -620,8 +603,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: dg[0].Name, Explicit: true, } - err = badTargetMod.validate(dg, testModToGrp) - c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["varNotFound"])) + c.Assert(badTargetMod.validate(dg), ErrorMatches, fmt.Sprintf("module bad-module was not found")) // A source group ID that doesn't exist (bad) badSourceGroup := modReference{ @@ -630,8 +612,7 @@ func (s *MySuite) TestValidateModuleReference(c *C) { FromGroupID: "bad-group", Explicit: true, } - err = badSourceGroup.validate(dg, testModToGrp) - c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["groupNotFound"])) + c.Assert(badSourceGroup.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["groupNotFound"])) } func (s *MySuite) TestIdentifySimpleVariable(c *C) { @@ -724,45 +705,36 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { groupIndex: 1, } - // strictly speaking this unit test now also tests this function - // we should expand the blueprint used in config_test.go to include - // multiple deployment groups - testModToGrp, err := checkModuleAndGroupNames(testBlueprint.DeploymentGroups) - c.Assert(err, IsNil) - c.Assert(testModToGrp[testModule0.ID], Equals, 0) - c.Assert(testModToGrp[testModule1.ID], Equals, 1) - // Invalid variable -> no . testVarContext1.varString = "$(varsStringWithNoDot)" - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err := expandSimpleVariable(testVarContext1) expectedErr := fmt.Sprintf("%s.*", errorMessages["invalidVar"]) c.Assert(err, ErrorMatches, expectedErr) // Global variable: Invalid -> not found testVarContext1.varString = "$(vars.doesntExists)" - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) expectedErr = fmt.Sprintf("%s: .*", errorMessages["varNotFound"]) c.Assert(err, ErrorMatches, expectedErr) // Global variable: Success testVarContext1.blueprint.Vars["globalExists"] = "existsValue" testVarContext1.varString = "$(vars.globalExists)" - got, err := expandSimpleVariable(testVarContext1, testModToGrp) + got, err := expandSimpleVariable(testVarContext1) c.Assert(err, IsNil) c.Assert(got, Equals, "((var.globalExists))") // Module variable: Invalid -> Module not found testVarContext1.varString = "$(bad_mod.someVar)" - _, err = expandSimpleVariable(testVarContext1, testModToGrp) - expectedErr = fmt.Sprintf("%s: .*", errorMessages["varNotFound"]) - c.Assert(err, ErrorMatches, expectedErr) + _, err = expandSimpleVariable(testVarContext1) + c.Assert(err, ErrorMatches, "module bad_mod was not found") // Module variable: Invalid -> Output not found reader := modulereader.Factory("terraform") reader.SetInfo(testModule1.Source, modulereader.ModuleInfo{}) fakeOutput := "doesntExist" testVarContext1.varString = fmt.Sprintf("$(%s.%s)", testModule1.ID, fakeOutput) - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) expectedErr = fmt.Sprintf("%s: module %s did not have output %s", errorMessages["noOutput"], testModule1.ID, fakeOutput) c.Assert(err, ErrorMatches, expectedErr) @@ -776,7 +748,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s)", testModule1.ID, existingOutput) - got, err = expandSimpleVariable(testVarContext1, testModToGrp) + got, err = expandSimpleVariable(testVarContext1) c.Assert(err, IsNil) expectedErr = fmt.Sprintf("((module.%s.%s))", testModule1.ID, existingOutput) c.Assert(got, Equals, expectedErr) @@ -790,7 +762,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[1].Name, testModule1.ID, existingOutput) - got, err = expandSimpleVariable(testVarContext1, testModToGrp) + got, err = expandSimpleVariable(testVarContext1) c.Assert(err, IsNil) c.Assert(got, Equals, fmt.Sprintf("((module.%s.%s))", testModule1.ID, existingOutput)) @@ -804,7 +776,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule1.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) c.Assert(err, NotNil) expectedErr = fmt.Sprintf("%s: %s.%s should be %s.%s", @@ -821,7 +793,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule0.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s)", testModule0.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) expectedErr = fmt.Sprintf("%s: %s .*", errorMessages["intergroupImplicit"], testModule0.ID) c.Assert(err, ErrorMatches, expectedErr) @@ -829,15 +801,14 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { // Intergroup variable: failure because explicit group and module does not exist testVarContext1.varString = fmt.Sprintf("$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, "bad_module", "bad_output") - _, err = expandSimpleVariable(testVarContext1, testModToGrp) - expectedErr = fmt.Sprintf("%s: .*", errorMessages["varNotFound"]) - c.Assert(err, ErrorMatches, expectedErr) + _, err = expandSimpleVariable(testVarContext1) + c.Assert(err, ErrorMatches, "module bad_module was not found") // Intergroup variable: failure because explicit group and output does not exist fakeOutput = "bad_output" testVarContext1.varString = fmt.Sprintf("$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule0.ID, fakeOutput) - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) expectedErr = fmt.Sprintf("%s: module %s did not have output %s", errorMessages["noOutput"], testModule0.ID, fakeOutput) c.Assert(err, ErrorMatches, expectedErr) @@ -850,7 +821,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext0.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[1].Name, testModule1.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext0, testModToGrp) + _, err = expandSimpleVariable(testVarContext0) expectedErr = fmt.Sprintf("%s: %s .*", errorMessages["intergroupOrder"], testModule1.ID) c.Assert(err, ErrorMatches, expectedErr) @@ -864,7 +835,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule0.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule0.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1, testModToGrp) + _, err = expandSimpleVariable(testVarContext1) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: %s .*", errorMessages["varInAnotherGroup"], regexp.QuoteMeta(testVarContext1.varString))) } @@ -872,8 +843,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { // presently it returns only an error func (s *MySuite) TestExpandVariable(c *C) { testVarContext0 := varContext{} - testModToGrp := make(map[string]int) - str, err := expandVariable(testVarContext0, testModToGrp) + str, err := expandVariable(testVarContext0) c.Assert(str, Equals, "") c.Assert(err, NotNil) } diff --git a/pkg/config/validator_test.go b/pkg/config/validator_test.go index 61cefa5c37..73714574ef 100644 --- a/pkg/config/validator_test.go +++ b/pkg/config/validator_test.go @@ -67,10 +67,9 @@ func (s *MySuite) TestValidateModuleSettings(c *C) { Modules: []Module{{Kind: "terraform", Source: testSource, Settings: testSettings}}, } dc := DeploymentConfig{ - Config: Blueprint{DeploymentGroups: []DeploymentGroup{testDeploymentGroup}}, - ModulesInfo: map[string]map[string]modulereader.ModuleInfo{}, - ModuleToGroup: map[string]int{}, - expanded: false, + Config: Blueprint{DeploymentGroups: []DeploymentGroup{testDeploymentGroup}}, + ModulesInfo: map[string]map[string]modulereader.ModuleInfo{}, + expanded: false, } dc.validateModuleSettings() } From 3a95521c1f831455e819d704be71b8d47c10a6c3 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 14 Mar 2023 14:19:29 -0500 Subject: [PATCH 010/100] Address feedback in #983 --- .../modules/scripts/htcondor-install/files/autoscaler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/community/modules/scripts/htcondor-install/files/autoscaler.py b/community/modules/scripts/htcondor-install/files/autoscaler.py index e4605d6099..e1392dadf5 100644 --- a/community/modules/scripts/htcondor-install/files/autoscaler.py +++ b/community/modules/scripts/htcondor-install/files/autoscaler.py @@ -185,10 +185,10 @@ def scale(self): # stewardship. A full list of Job ClassAd attributes can be found at # https://htcondor.readthedocs.io/en/latest/classad-attributes/job-classad-attributes.html schedd = htcondor.Schedd() - # encourage the job queue to start a new negotiation cycle - # there are internal unconfigurable rate limits so not guaranteed + # encourage the job queue to start a new negotiation cycle; there are + # internal unconfigurable rate limits so not guaranteed; this is not + # strictly required for success, but may reduce latency of autoscaling schedd.reschedule() - time.sleep(3) REQUEST_CPUS_ATTRIBUTE = "RequestCpus" REQUEST_GPUS_ATTRIBUTE = "RequestGpus" REQUEST_MEMORY_ATTRIBUTE = "RequestMemory" From 34645bd371fcb56bbb3cf5c798b1c8503f8751b8 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Tue, 14 Mar 2023 15:44:54 -0700 Subject: [PATCH 011/100] Fix broken markfown link (#1024) Co-authored-by: Ivan Orlov --- .../modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md index 9533805ac2..690194cdd7 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md @@ -78,7 +78,7 @@ This option has some additional requirements: pip3 install -r requirements.txt --user ``` - For more information, see the [description](#optdeps) of this module. + For more information, see the [description][optdeps] of this module. * The project in your gcloud config must match the project the cluster is being deployed onto due to a known issue with the reconfigure scripts. To set your default config project, run the following command: From 6cf753935f2a1f66737b75d3a5c505080f6b0242 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 14 Mar 2023 22:21:25 -0500 Subject: [PATCH 012/100] Update year of Terraform deployment license --- pkg/modulewriter/license.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/modulewriter/license.go b/pkg/modulewriter/license.go index 86b8f0c75d..b214e92ed0 100644 --- a/pkg/modulewriter/license.go +++ b/pkg/modulewriter/license.go @@ -17,7 +17,7 @@ package modulewriter const license string = `/** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From fb100a95cd092ef5058c46ef56d38897eec05788 Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Wed, 15 Mar 2023 09:21:54 -0700 Subject: [PATCH 013/100] Fix: dependabot proposed incompatible requirements --- community/front-end/ofe/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 9641a3605f..80dac6fc6f 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -1,7 +1,7 @@ archspec==0.1.3 argcomplete==2.0.0 asgiref==3.5.0 -astroid==2.15.0 +astroid==2.9.3 cachetools==5.0.0 certifi==2022.12.7 cffi==1.15.0 @@ -29,7 +29,7 @@ google-resumable-media==2.2.1 googleapis-common-protos==1.58.0 grafana_api==1.0.3 grpc-google-iam-v1==0.12.3 -grpcio==1.43.0 +grpcio==1.51.3 grpcio-status==1.51.3 h11==0.13.0 httplib2==0.21.0 @@ -43,7 +43,7 @@ oauthlib==3.2.2 platformdirs==2.5.0 pre-commit==2.17.0 proto-plus==1.22.2 -protobuf==3.19.5 +protobuf==4.22.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 @@ -67,6 +67,6 @@ typing_extensions==4.1.1 uritemplate==4.1.1 urllib3==1.26.8 uvicorn==0.17.4 -wrapt==1.15.0 +wrapt==1.13.3 xmltodict==0.12.0 yq==2.13.0 From 8dce57ce60c3750a14e4d6e47e79d76b5c183de1 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Wed, 15 Mar 2023 15:25:26 -0700 Subject: [PATCH 014/100] Fix startup-options test (#1029) --- tools/validate_configs/test_configs/startup-options.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/validate_configs/test_configs/startup-options.yaml b/tools/validate_configs/test_configs/startup-options.yaml index 94a22d5ae1..be907d67d5 100644 --- a/tools/validate_configs/test_configs/startup-options.yaml +++ b/tools/validate_configs/test_configs/startup-options.yaml @@ -67,7 +67,6 @@ deployment_groups: settings: name_prefix: use-startup machine_type: e2-standard-4 - startup_script: $(startup.startup_script) - id: instance-metadata-startup source: ./modules/compute/vm-instance From 369dc717e0098e30d3786f97ce96cb6df748ee93 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Wed, 15 Mar 2023 15:53:13 -0700 Subject: [PATCH 015/100] Fix broken test config use-resources.yaml (#1030) ``` $ ./ghpc create tools/validate_configs/test_configs/use-resources.yaml -l ERROR --vars="project_id=io-playground" module homefs uses module network1, but matching setting and outputs were not found. This may be because the value is set explicitly or set by a prior used module module slurm_login uses module homefs, but matching setting and outputs were not found. This may be because the value is set explicitly or set by a prior used module module slurm_login uses module scratchfs, but matching setting and outputs were not found. This may be because the value is set explicitly or set by a prior used module module slurm_login uses module projectsfs, but matching setting and outputs were not found. This may be because the value is set explicitly or set by a prior used module One or more used modules could not have their settings and outputs linked. error: validator test_module_not_used failed ``` --- tools/validate_configs/test_configs/use-resources.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/validate_configs/test_configs/use-resources.yaml b/tools/validate_configs/test_configs/use-resources.yaml index ab18d1bbc7..d2e39eeeb2 100644 --- a/tools/validate_configs/test_configs/use-resources.yaml +++ b/tools/validate_configs/test_configs/use-resources.yaml @@ -36,7 +36,6 @@ deployment_groups: use: [network1] settings: local_mount: /home - network_id: $(network1.network_id) - id: projectsfs @@ -78,4 +77,3 @@ deployment_groups: - network1 settings: login_machine_type: n2-standard-4 - network_storage: $(projectsfs.network_storage) From 7b34c6abaff9de620e46dc8f83fadddb6c27b5f6 Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Tue, 14 Mar 2023 16:53:58 -0700 Subject: [PATCH 016/100] Add an integration test for chrome-remote-desktop module --- .../daily-tests/integration-group-4.yaml | 25 ++++++++++++++++++- .../tests/chrome-remote-desktop.yml | 24 ++++++++++++++++++ .../test_configs/remote-desktop.yaml | 4 +-- 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml diff --git a/tools/cloud-build/daily-tests/integration-group-4.yaml b/tools/cloud-build/daily-tests/integration-group-4.yaml index b97792bfd1..89dae084db 100644 --- a/tools/cloud-build/daily-tests/integration-group-4.yaml +++ b/tools/cloud-build/daily-tests/integration-group-4.yaml @@ -20,7 +20,7 @@ # └── htcondor (group 4) # └── Cloud Batch # └── slurm-gcp-v5-ubuntu2004 - +# └── chrome-remote-desktop timeout: 14400s # 4hr steps: @@ -107,3 +107,26 @@ steps: ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-v5-ubuntu.yml" + +## Test chrome-remote-desktop module +- id: chrome-remote-desktop + waitFor: + - slurm-gcp-v5-ubuntu2004 + - fetch_builder + - build_ghpc + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + env: + - "ANSIBLE_HOST_KEY_CHECKING=false" + - "ANSIBLE_CONFIG=/workspace/tools/cloud-build/ansible.cfg" + args: + - -c + - | + set -x -e + BUILD_ID_FULL=$BUILD_ID + BUILD_ID_SHORT=$${BUILD_ID_FULL:0:6} + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \ + --user=sa_106486320838376751393 --extra-vars="project=${PROJECT_ID} build=$${BUILD_ID_SHORT}" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml" diff --git a/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml b/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml new file mode 100644 index 0000000000..e79ddfe876 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml @@ -0,0 +1,24 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +test_name: chrome-remote-desktop +deployment_name: chrome-remote-desktop-{{ build }} +zone: us-central1-c +workspace: /workspace +blueprint_yaml: "{{ workspace }}/tools/validate_configs/test_configs/remote-desktop.yaml" +network: "{{ deployment_name }}-net" +remote_node: "{{ deployment_name }}-0" +post_deploy_tests: [] +custom_vars: + project: "{{ project }}" diff --git a/tools/validate_configs/test_configs/remote-desktop.yaml b/tools/validate_configs/test_configs/remote-desktop.yaml index bf42911271..f18b6d0902 100644 --- a/tools/validate_configs/test_configs/remote-desktop.yaml +++ b/tools/validate_configs/test_configs/remote-desktop.yaml @@ -19,8 +19,8 @@ blueprint_name: remote-desktop vars: project_id: ## Set GCP Project ID Here ## deployment_name: remote-desktop - region: us-east4 - zone: us-east4-c + region: us-central1 + zone: us-central1-c deployment_groups: - group: primary From 5948dc9a7c68c4e27914b362b9db856bcc09a8af Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Tue, 14 Mar 2023 17:12:42 -0700 Subject: [PATCH 017/100] Add retries to apt tasks in chrome-remote-desktop to account for lock contention with unattended-upgrades --- .../scripts/configure-chrome-desktop.yml | 8 ++++++++ .../scripts/configure-grid-drivers.yml | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml b/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml index f8baccab01..22c9096834 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml +++ b/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-chrome-desktop.yml @@ -24,6 +24,10 @@ - xfce4-goodies state: present update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success - name: Download and configure CRD ansible.builtin.get_url: @@ -36,6 +40,10 @@ deb: /tmp/chrome-remote-desktop_current_amd64.deb environment: DEBIAN_FRONTEND: noninteractive + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success - name: Configure CRD to use Xfce by default ansible.builtin.copy: diff --git a/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml b/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml index e6eb5b49fd..203a56822d 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml +++ b/community/modules/remote-desktop/chrome-remote-desktop/scripts/configure-grid-drivers.yml @@ -32,6 +32,10 @@ - gdm3 state: present update_cache: true + register: apt_result + retries: 6 + delay: 10 + until: apt_result is success - name: Download GPU driver ansible.builtin.get_url: From f8e96b1cb29ab122dd3c98ee08c69ed91453c3e3 Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Thu, 16 Mar 2023 09:21:30 -0700 Subject: [PATCH 018/100] Bring all provider_meta versions up to current version --- .../scheduler/schedmd-slurm-gcp-v5-controller/versions.tf | 2 +- .../modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf index 688023fa24..3304d1f4a9 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.6.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.14.1" } required_version = ">= 0.14.0" } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf index 0489881fc7..cd1baf2540 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.6.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.14.1" } required_version = ">= 0.14.0" } From c35748ff77a2a695eb87d872932ffe275577b68e Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Thu, 16 Mar 2023 12:11:13 -0500 Subject: [PATCH 019/100] Update pre-commit hook repos --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f3194313c1..2ef20884e0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ --- repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.77.0 + rev: v1.77.1 hooks: - id: terraform_fmt - id: terraform_tflint @@ -66,7 +66,7 @@ repos: hooks: - id: go-critic args: [-disable, "#experimental,sloppyTypeAssert"] -- repo: https://github.com/ansible/ansible-lint.git +- repo: https://github.com/ansible/ansible-lint rev: v6.11.0 hooks: - id: ansible-lint @@ -75,13 +75,13 @@ repos: types: [yaml] additional_dependencies: - ansible==6.* -- repo: https://github.com/adrienverge/yamllint.git +- repo: https://github.com/adrienverge/yamllint rev: v1.29.0 hooks: - id: yamllint args: [-c=.yamllint] - repo: https://github.com/jackdewinter/pymarkdown - rev: v0.9.8 + rev: v0.9.9 hooks: - id: pymarkdown # Rules at https://github.com/jackdewinter/pymarkdown/tree/main/docs/rules From 91f3612cdae78155f19c38fa17a60aa1088293ae Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Thu, 16 Mar 2023 12:15:29 -0700 Subject: [PATCH 020/100] Add integration test coverage for add_deployment_name_before_prefix --- .../cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml b/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml index 420e36fec2..cedc782852 100644 --- a/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml +++ b/tools/cloud-build/daily-tests/blueprints/lustre-with-new-vpc.yaml @@ -63,6 +63,7 @@ deployment_groups: - mount-exascaler settings: name_prefix: test-workstation1 + add_deployment_name_before_prefix: true machine_type: c2-standard-4 - id: wait0 @@ -93,6 +94,7 @@ deployment_groups: - mount-exascaler-from-pre-existing settings: name_prefix: test-workstation2 + add_deployment_name_before_prefix: false machine_type: n2-standard-4 - id: wait1 From 71cd8595af29c4180f6033edc67295fb652af271 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Thu, 16 Mar 2023 17:21:55 +0000 Subject: [PATCH 021/100] Fix dtype of slurm node-group.preemtible to bool --- .../modules/compute/schedmd-slurm-gcp-v5-node-group/README.md | 2 +- .../compute/schedmd-slurm-gcp-v5-node-group/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md index eb1af44ec6..d51bb97a35 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md @@ -152,7 +152,7 @@ No modules. | [node\_count\_dynamic\_max](#input\_node\_count\_dynamic\_max) | Maximum number of dynamic nodes allowed in this partition. | `number` | `10` | no | | [node\_count\_static](#input\_node\_count\_static) | Number of nodes to be statically created. | `number` | `0` | no | | [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy.

Note: Placement groups are not supported when on\_host\_maintenance is set to
"MIGRATE" and will be deactivated regardless of the value of
enable\_placement. To support enable\_placement, ensure on\_host\_maintenance is
set to "TERMINATE". | `string` | `"TERMINATE"` | no | -| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `string` | `false` | no | +| [preemptible](#input\_preemptible) | Should use preemptibles to burst. | `bool` | `false` | no | | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes | | [service\_account](#input\_service\_account) | Service account to attach to the compute instances. If not set, the
default compute service account for the given project will be used with the
"https://www.googleapis.com/auth/cloud-platform" scope. | 
object({
    email  = string
    scopes = set(string)
  })
| `null` | no | | [shielded\_instance\_config](#input\_shielded\_instance\_config) | Shielded VM configuration for the instance. Note: not used unless
enable\_shielded\_vm is 'true'.
- enable\_integrity\_monitoring : Compare the most recent boot measurements to the
integrity policy baseline and return a pair of pass/fail results depending on
whether they match or not.
- enable\_secure\_boot : Verify the digital signature of all boot components, and
halt the boot process if signature verification fails.
- enable\_vtpm : Use a virtualized trusted platform module, which is a
specialized computer chip you can use to encrypt objects like keys and
certificates. | 
object({
    enable_integrity_monitoring = bool
    enable_secure_boot          = bool
    enable_vtpm                 = bool
  })
| 
{
  "enable_integrity_monitoring": true,
  "enable_secure_boot": true,
  "enable_vtpm": true
}
| no | diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf index 026d7bdaec..86ad5b9e69 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf @@ -269,7 +269,7 @@ variable "guest_accelerator" { variable "preemptible" { description = "Should use preemptibles to burst." - type = string + type = bool default = false } From 2f02b77d389b39ae67cd5e32bcd75a723795f217 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Thu, 16 Mar 2023 20:13:33 +0000 Subject: [PATCH 022/100] Set label dtype = map(string) for community/modules Fix type inconsistency ``` name: labels type: any decalred: nfs-server cloud-storage-bucket DDN-EXAScaler pbspro-execution schedmd-slurm-gcp-v5-node-group SchedMD-slurm-on-gcp-partition chrome-remote-desktop slurm-cloudsql-federation SchedMD-slurm-on-gcp-controller SchedMD-slurm-on-gcp-login-node pbspro-server pbspro-client type: map(string) decalred: startup-script filestore vm-instance batch-login-node batch-job-template htcondor-execute-point schedmd-slurm-gcp-v5-controller schedmd-slurm-gcp-v5-login htcondor-configure new-project ``` --- .../compute/SchedMD-slurm-on-gcp-partition/README.md | 2 +- .../compute/SchedMD-slurm-on-gcp-partition/variables.tf | 4 ++-- community/modules/compute/pbspro-execution/README.md | 2 +- community/modules/compute/pbspro-execution/variables.tf | 4 ++-- .../compute/schedmd-slurm-gcp-v5-node-group/README.md | 2 +- .../compute/schedmd-slurm-gcp-v5-node-group/variables.tf | 4 ++-- .../modules/database/slurm-cloudsql-federation/README.md | 2 +- .../modules/database/slurm-cloudsql-federation/variables.tf | 4 ++-- community/modules/file-system/DDN-EXAScaler/README.md | 2 +- community/modules/file-system/DDN-EXAScaler/variables.tf | 4 ++-- .../modules/file-system/cloud-storage-bucket/README.md | 2 +- .../modules/file-system/cloud-storage-bucket/variables.tf | 4 ++-- community/modules/file-system/nfs-server/README.md | 2 +- community/modules/file-system/nfs-server/variables.tf | 4 ++-- .../modules/remote-desktop/chrome-remote-desktop/README.md | 2 +- .../remote-desktop/chrome-remote-desktop/variables.tf | 6 +++--- .../scheduler/SchedMD-slurm-on-gcp-controller/README.md | 2 +- .../scheduler/SchedMD-slurm-on-gcp-controller/variables.tf | 4 ++-- .../scheduler/SchedMD-slurm-on-gcp-login-node/README.md | 2 +- .../scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf | 4 ++-- community/modules/scheduler/pbspro-client/README.md | 2 +- community/modules/scheduler/pbspro-client/variables.tf | 4 ++-- community/modules/scheduler/pbspro-server/README.md | 2 +- community/modules/scheduler/pbspro-server/variables.tf | 4 ++-- modules/scripts/startup-script/README.md | 2 +- modules/scripts/startup-script/variables.tf | 2 +- 26 files changed, 39 insertions(+), 39 deletions(-) diff --git a/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md b/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md index 9b1b414326..d1c6d5db1c 100644 --- a/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md +++ b/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md @@ -74,7 +74,7 @@ No resources. | [image\_hyperthreads](#input\_image\_hyperthreads) | Enable hyperthreading | `bool` | `false` | no | | [instance\_image](#input\_instance\_image) | Defines the image that will be used by the compute VMs in this partition.
Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.
Custom images must comply with Slurm on GCP requirements. | `map(string)` | 
{
  "family": "schedmd-slurm-21-08-8-hpc-centos-7",
  "project": "schedmd-slurm-public"
}
| no | | [instance\_template](#input\_instance\_template) | Instance template to use to create partition instances | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. List of key key, value pairs. | `any` | `{}` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | | [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes | `string` | `"c2-standard-60"` | no | | [max\_node\_count](#input\_max\_node\_count) | Maximum number of nodes allowed in this partition | `number` | `50` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | diff --git a/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf b/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf index c679a960fc..ef21cda036 100644 --- a/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf +++ b/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf @@ -86,8 +86,8 @@ variable "compute_disk_size_gb" { } variable "labels" { - description = "Labels to add to partition compute instances. List of key key, value pairs." - type = any + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) default = {} } diff --git a/community/modules/compute/pbspro-execution/README.md b/community/modules/compute/pbspro-execution/README.md index 84d275144e..96107e8a05 100644 --- a/community/modules/compute/pbspro-execution/README.md +++ b/community/modules/compute/pbspro-execution/README.md @@ -96,7 +96,7 @@ No resources. | [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `null` | no | | [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | | [instance\_image](#input\_instance\_image) | Instance Image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | | [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | | [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | diff --git a/community/modules/compute/pbspro-execution/variables.tf b/community/modules/compute/pbspro-execution/variables.tf index 774f63e8b6..f7bda1d4b1 100644 --- a/community/modules/compute/pbspro-execution/variables.tf +++ b/community/modules/compute/pbspro-execution/variables.tf @@ -137,8 +137,8 @@ variable "machine_type" { } variable "labels" { - description = "Labels to add to the instances. List key, value pairs." - type = any + description = "Labels to add to the instances. Key-value pairs." + type = map(string) } variable "service_account" { diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md index d51bb97a35..1c7f8d4c6a 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md @@ -143,7 +143,7 @@ No modules. | [guest\_accelerator](#input\_guest\_accelerator) | Alternative method of providing 'var.gpu' with a consistent naming scheme to
other HPC Toolkit modules.

If both 'var.gpu' and 'var.guest\_accelerator' are set, 'var.gpu' will be used. | 
list(object({
    type  = string,
    count = number
  }))
| `null` | no | | [instance\_image](#input\_instance\_image) | Defines the image that will be used in the node group VM instances. This
value is overridden if any of `source_image`, `source_image_family` or
`source_image_project` are set.

Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.

For more information on creating custom images that comply with Slurm on GCP
see the "Slurm on GCP Custom Images" section in docs/vm-images.md. | `map(string)` | 
{
  "family": "schedmd-v5-slurm-22-05-8-hpc-centos-7",
  "project": "projects/schedmd-slurm-public/global/images/family"
}
| no | | [instance\_template](#input\_instance\_template) | Self link to a custom instance template. If set, other VM definition
variables such as machine\_type and instance\_image will be ignored in favor
of the provided instance template.

For more information on creating custom images for the instance template
that comply with Slurm on GCP see the "Slurm on GCP Custom Images" section
in docs/vm-images.md. | `string` | `null` | no | -| [labels](#input\_labels) | Labels to add to partition compute instances. List of key key, value pairs. | `any` | `{}` | no | +| [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | | [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes. | `string` | `"c2-standard-60"` | no | | [metadata](#input\_metadata) | Metadata, provided as a map. | `map(string)` | `{}` | no | | [min\_cpu\_platform](#input\_min\_cpu\_platform) | The name of the minimum CPU platform that you want the instance to use. | `string` | `null` | no | diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf index 86ad5b9e69..2f7e345c2e 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf @@ -213,8 +213,8 @@ variable "enable_smt" { } variable "labels" { - description = "Labels to add to partition compute instances. List of key key, value pairs." - type = any + description = "Labels to add to partition compute instances. Key-value pairs." + type = map(string) default = {} } diff --git a/community/modules/database/slurm-cloudsql-federation/README.md b/community/modules/database/slurm-cloudsql-federation/README.md index 7ca93be525..2af5325323 100644 --- a/community/modules/database/slurm-cloudsql-federation/README.md +++ b/community/modules/database/slurm-cloudsql-federation/README.md @@ -77,7 +77,7 @@ No modules. |------|-------------|------|---------|:--------:| | [deletion\_protection](#input\_deletion\_protection) | Whether or not to allow Terraform to destroy the instance. | `string` | `false` | no | | [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the instances. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | | [nat\_ips](#input\_nat\_ips) | a list of NAT ips to be allow listed for the slurm cluster communication | `list(any)` | n/a | yes | | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | | [region](#input\_region) | The region where SQL instance will be configured | `string` | n/a | yes | diff --git a/community/modules/database/slurm-cloudsql-federation/variables.tf b/community/modules/database/slurm-cloudsql-federation/variables.tf index cc7cd813ad..806568607f 100644 --- a/community/modules/database/slurm-cloudsql-federation/variables.tf +++ b/community/modules/database/slurm-cloudsql-federation/variables.tf @@ -51,8 +51,8 @@ variable "deletion_protection" { } variable "labels" { - description = "Labels to add to the instances. List key, value pairs." - type = any + description = "Labels to add to the instances. Key-value pairs." + type = map(string) } variable "sql_username" { diff --git a/community/modules/file-system/DDN-EXAScaler/README.md b/community/modules/file-system/DDN-EXAScaler/README.md index 0a87dfa7b7..bc723a185c 100644 --- a/community/modules/file-system/DDN-EXAScaler/README.md +++ b/community/modules/file-system/DDN-EXAScaler/README.md @@ -115,7 +115,7 @@ No resources. | [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | | [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | | [image](#input\_image) | Source image properties | `any` | 
{
  "family": "exascaler-cloud-6-1-centos",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. List of key key, value pairs. | `any` | `{}` | no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(any)` | `{}` | no | | [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | | [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | | [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | diff --git a/community/modules/file-system/DDN-EXAScaler/variables.tf b/community/modules/file-system/DDN-EXAScaler/variables.tf index c9a12771e1..1cf61be22f 100644 --- a/community/modules/file-system/DDN-EXAScaler/variables.tf +++ b/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -462,7 +462,7 @@ variable "prefix" { } variable "labels" { - description = "Labels to add to EXAScaler Cloud deployment. List of key key, value pairs." - type = any + description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." + type = map(any) default = {} } diff --git a/community/modules/file-system/cloud-storage-bucket/README.md b/community/modules/file-system/cloud-storage-bucket/README.md index 477224e68a..465114b796 100644 --- a/community/modules/file-system/cloud-storage-bucket/README.md +++ b/community/modules/file-system/cloud-storage-bucket/README.md @@ -134,7 +134,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the GCS bucket. | `string` | n/a | yes | -| [labels](#input\_labels) | Labels to add to the GCS bucket. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | | [local\_mount](#input\_local\_mount) | The mount point where the contents of the device may be accessed after mounting. | `string` | `"/mnt"` | no | | [mount\_options](#input\_mount\_options) | Mount options to be put in fstab. Note: `implicit_dirs` makes it easier to work with objects added by other tools, but there is a performance impact. See: [more information](https://github.com/GoogleCloudPlatform/gcsfuse/blob/master/docs/semantics.md#implicit-directories) | `string` | `"defaults,_netdev,implicit_dirs"` | no | | [name\_prefix](#input\_name\_prefix) | Name Prefix. | `string` | `null` | no | diff --git a/community/modules/file-system/cloud-storage-bucket/variables.tf b/community/modules/file-system/cloud-storage-bucket/variables.tf index 844ff3e65c..ece08811c5 100644 --- a/community/modules/file-system/cloud-storage-bucket/variables.tf +++ b/community/modules/file-system/cloud-storage-bucket/variables.tf @@ -30,8 +30,8 @@ variable "region" { } variable "labels" { - description = "Labels to add to the GCS bucket. List key, value pairs." - type = any + description = "Labels to add to the GCS bucket. Key-value pairs." + type = map(string) } variable "local_mount" { diff --git a/community/modules/file-system/nfs-server/README.md b/community/modules/file-system/nfs-server/README.md index f1168ec832..4a29934884 100644 --- a/community/modules/file-system/nfs-server/README.md +++ b/community/modules/file-system/nfs-server/README.md @@ -121,7 +121,7 @@ No modules. | [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used as name of the NFS instace if no name is specified. | `string` | n/a | yes | | [disk\_size](#input\_disk\_size) | Storage size gb | `number` | `"100"` | no | | [image](#input\_image) | the VM image used by the nfs server | `string` | `"cloud-hpc-image-public/hpc-centos-7"` | no | -| [labels](#input\_labels) | Labels to add to the NFS instance. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the NFS instance. Key-value pairs. | `map(string)` | n/a | yes | | [local\_mounts](#input\_local\_mounts) | Mountpoint for this NFS compute instance | `list(string)` | 
[
  "/data"
]
| no | | [machine\_type](#input\_machine\_type) | Type of the VM instance to use | `string` | `"n2d-standard-2"` | no | | [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | diff --git a/community/modules/file-system/nfs-server/variables.tf b/community/modules/file-system/nfs-server/variables.tf index 2aad522d38..809093b82b 100644 --- a/community/modules/file-system/nfs-server/variables.tf +++ b/community/modules/file-system/nfs-server/variables.tf @@ -78,8 +78,8 @@ variable "machine_type" { } variable "labels" { - description = "Labels to add to the NFS instance. List key, value pairs." - type = any + description = "Labels to add to the NFS instance. Key-value pairs." + type = map(string) } variable "metadata" { diff --git a/community/modules/remote-desktop/chrome-remote-desktop/README.md b/community/modules/remote-desktop/chrome-remote-desktop/README.md index c0d34a28f0..448fc6c95f 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/README.md +++ b/community/modules/remote-desktop/chrome-remote-desktop/README.md @@ -85,7 +85,7 @@ No resources. | [install\_nvidia\_driver](#input\_install\_nvidia\_driver) | Installs the nvidia driver (true/false). For details, see https://cloud.google.com/compute/docs/gpus/install-drivers-gpu | `bool` | n/a | yes | | [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | | [instance\_image](#input\_instance\_image) | Instance Image. An alternative could be family = "ubuntu-2004-lts" and project = "ubuntu-os-cloud" or family = "debian-11" and project = "debian-cloud" | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "ubuntu-2204-lts",
  "project": "ubuntu-os-cloud"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. List key, value pairs. | `any` | `[]` | no | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | `{}` | no | | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | | [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | | [name\_prefix](#input\_name\_prefix) | Name Prefix | `string` | `null` | no | diff --git a/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/community/modules/remote-desktop/chrome-remote-desktop/variables.tf index 03c8fb7808..e51e4efd1a 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ b/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -104,9 +104,9 @@ variable "machine_type" { } variable "labels" { - description = "Labels to add to the instances. List key, value pairs." - type = any - default = [] + description = "Labels to add to the instances. Key-value pairs." + type = map(string) + default = {} } variable "service_account" { diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md index 97208af4e8..d4abf9ce7f 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md @@ -109,7 +109,7 @@ No resources. | [instance\_image](#input\_instance\_image) | Slurm image to use for the controller instance.
Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.
Custom images must comply with Slurm on GCP requirements. | `map(string)` | 
{
  "family": "schedmd-slurm-21-08-8-hpc-centos-7",
  "project": "schedmd-slurm-public"
}
| no | | [intel\_select\_solution](#input\_intel\_select\_solution) | Configure the cluster to meet the performance requirement of the Intel Select Solution | `string` | `null` | no | | [jwt\_key](#input\_jwt\_key) | Specific libjwt key to use | `any` | `null` | no | -| [labels](#input\_labels) | Labels to add to controller instance. List of key key, value pairs. | `any` | `{}` | no | +| [labels](#input\_labels) | Labels to add to controller instance. Key-value pairs. | `map(string)` | `{}` | no | | [login\_node\_count](#input\_login\_node\_count) | Number of login nodes in the cluster | `number` | `0` | no | | [munge\_key](#input\_munge\_key) | Specific munge key to use | `any` | `null` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf index d40e1e3476..492a4e09a2 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf @@ -105,8 +105,8 @@ variable "disable_compute_public_ips" { } variable "labels" { - description = "Labels to add to controller instance. List of key key, value pairs." - type = any + description = "Labels to add to controller instance. Key-value pairs." + type = map(string) default = {} } diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md index 3923e5c7fc..aca6750e3b 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md @@ -97,7 +97,7 @@ No resources. | [deployment\_name](#input\_deployment\_name) | Name of the deployment | `string` | n/a | yes | | [disable\_login\_public\_ips](#input\_disable\_login\_public\_ips) | If set to true, create Cloud NAT gateway and enable IAP FW rules | `bool` | `false` | no | | [instance\_image](#input\_instance\_image) | Disk OS image with Slurm preinstalled to use for login node.
Expected Fields:
name: The name of the image. Mutually exclusive with family.
family: The image family to use. Mutually exclusive with name.
project: The project where the image is hosted.
Custom images must comply with Slurm on GCP requirements. | `map(string)` | 
{
  "family": "schedmd-slurm-21-08-8-hpc-centos-7",
  "project": "schedmd-slurm-public"
}
| no | -| [labels](#input\_labels) | Labels to add to login instances. List of key key, value pairs. | `any` | `{}` | no | +| [labels](#input\_labels) | Labels to add to login instances. Key-value pairs. | `map(string)` | `{}` | no | | [login\_instance\_template](#input\_login\_instance\_template) | Instance template to use to create controller instance | `string` | `null` | no | | [login\_machine\_type](#input\_login\_machine\_type) | Machine type to use for login node instances. | `string` | `"n2-standard-2"` | no | | [login\_node\_count](#input\_login\_node\_count) | Number of login nodes in the cluster | `number` | `1` | no | diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf index 3a34cecd40..d4e455a2ca 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf @@ -87,8 +87,8 @@ variable "disable_login_public_ips" { } variable "labels" { - description = "Labels to add to login instances. List of key key, value pairs." - type = any + description = "Labels to add to login instances. Key-value pairs." + type = map(string) default = {} } diff --git a/community/modules/scheduler/pbspro-client/README.md b/community/modules/scheduler/pbspro-client/README.md index 49c88e9e87..dcde7bfd6b 100644 --- a/community/modules/scheduler/pbspro-client/README.md +++ b/community/modules/scheduler/pbspro-client/README.md @@ -96,7 +96,7 @@ No resources. | [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `null` | no | | [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | | [instance\_image](#input\_instance\_image) | Instance Image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | | [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | | [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | diff --git a/community/modules/scheduler/pbspro-client/variables.tf b/community/modules/scheduler/pbspro-client/variables.tf index ecd21e3d44..7bb0141ce2 100644 --- a/community/modules/scheduler/pbspro-client/variables.tf +++ b/community/modules/scheduler/pbspro-client/variables.tf @@ -137,8 +137,8 @@ variable "machine_type" { } variable "labels" { - description = "Labels to add to the instances. List key, value pairs." - type = any + description = "Labels to add to the instances. Key-value pairs." + type = map(string) } variable "service_account" { diff --git a/community/modules/scheduler/pbspro-server/README.md b/community/modules/scheduler/pbspro-server/README.md index ccb263f12c..dbfd91b8bb 100644 --- a/community/modules/scheduler/pbspro-server/README.md +++ b/community/modules/scheduler/pbspro-server/README.md @@ -96,7 +96,7 @@ No resources. | [guest\_accelerator](#input\_guest\_accelerator) | List of the type and count of accelerator cards attached to the instance. | 
list(object({
    type  = string,
    count = number
  }))
| `null` | no | | [instance\_count](#input\_instance\_count) | Number of instances | `number` | `1` | no | | [instance\_image](#input\_instance\_image) | Instance Image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | -| [labels](#input\_labels) | Labels to add to the instances. List key, value pairs. | `any` | n/a | yes | +| [labels](#input\_labels) | Labels to add to the instances. Key-value pairs. | `map(string)` | n/a | yes | | [local\_ssd\_count](#input\_local\_ssd\_count) | The number of local SSDs to attach to each VM. See https://cloud.google.com/compute/docs/disks/local-ssd. | `number` | `0` | no | | [local\_ssd\_interface](#input\_local\_ssd\_interface) | Interface to be used with local SSDs. Can be either 'NVME' or 'SCSI'. No effect unless `local_ssd_count` is also set. | `string` | `"NVME"` | no | | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | diff --git a/community/modules/scheduler/pbspro-server/variables.tf b/community/modules/scheduler/pbspro-server/variables.tf index 060e02d79d..a4eccd2619 100644 --- a/community/modules/scheduler/pbspro-server/variables.tf +++ b/community/modules/scheduler/pbspro-server/variables.tf @@ -165,8 +165,8 @@ variable "machine_type" { } variable "labels" { - description = "Labels to add to the instances. List key, value pairs." - type = any + description = "Labels to add to the instances. Key-value pairs." + type = map(string) } variable "service_account" { diff --git a/modules/scripts/startup-script/README.md b/modules/scripts/startup-script/README.md index cac87e1351..af68492a6c 100644 --- a/modules/scripts/startup-script/README.md +++ b/modules/scripts/startup-script/README.md @@ -238,7 +238,7 @@ No modules. | [debug\_file](#input\_debug\_file) | Path to an optional local to be written with 'startup\_script'. | `string` | `null` | no | | [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment, used to name GCS bucket for startup scripts. | `string` | n/a | yes | | [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object. | `string` | `null` | no | -| [labels](#input\_labels) | Labels for the created GCS bucket. List key, value pairs. | `map(string)` | n/a | yes | +| [labels](#input\_labels) | Labels for the created GCS bucket. Key-value pairs. | `map(string)` | n/a | yes | | [prepend\_ansible\_installer](#input\_prepend\_ansible\_installer) | Prepend Ansible installation script if any of the specified runners are of type ansible-local | `bool` | `true` | no | | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | | [region](#input\_region) | The region to deploy to | `string` | n/a | yes | diff --git a/modules/scripts/startup-script/variables.tf b/modules/scripts/startup-script/variables.tf index acb67c61cc..93b5d6761f 100644 --- a/modules/scripts/startup-script/variables.tf +++ b/modules/scripts/startup-script/variables.tf @@ -43,7 +43,7 @@ variable "debug_file" { } variable "labels" { - description = "Labels for the created GCS bucket. List key, value pairs." + description = "Labels for the created GCS bucket. Key-value pairs." type = map(string) } From 757cb3b0af123f49da2eb3d23542dd0b1000dfb2 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Thu, 16 Mar 2023 22:51:35 +0000 Subject: [PATCH 023/100] Fix labels dtype in DDN-EXAScaler Miss out from PR #1038 --- community/modules/file-system/DDN-EXAScaler/README.md | 2 +- community/modules/file-system/DDN-EXAScaler/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community/modules/file-system/DDN-EXAScaler/README.md b/community/modules/file-system/DDN-EXAScaler/README.md index bc723a185c..233499b109 100644 --- a/community/modules/file-system/DDN-EXAScaler/README.md +++ b/community/modules/file-system/DDN-EXAScaler/README.md @@ -115,7 +115,7 @@ No resources. | [clt](#input\_clt) | Compute client target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 0,
  "disk_size": 256,
  "disk_type": "pd-standard"
}
| no | | [fsname](#input\_fsname) | EXAScaler filesystem name, only alphanumeric characters are allowed, and the value must be 1-8 characters long | `string` | `"exacloud"` | no | | [image](#input\_image) | Source image properties | `any` | 
{
  "family": "exascaler-cloud-6-1-centos",
  "project": "ddn-public"
}
| no | -| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(any)` | `{}` | no | +| [labels](#input\_labels) | Labels to add to EXAScaler Cloud deployment. Key-value pairs. | `map(string)` | `{}` | no | | [local\_mount](#input\_local\_mount) | Mountpoint (at the client instances) for this EXAScaler system | `string` | `"/shared"` | no | | [mds](#input\_mds) | Metadata server properties | 
object({
    node_type  = string
    node_cpu   = string
    nic_type   = string
    node_count = number
    public_ip  = bool
  })
| 
{
  "nic_type": "GVNIC",
  "node_count": 1,
  "node_cpu": "Intel Cascade Lake",
  "node_type": "n2-standard-32",
  "public_ip": true
}
| no | | [mdt](#input\_mdt) | Metadata target properties | 
object({
    disk_bus   = string
    disk_type  = string
    disk_size  = number
    disk_count = number
    disk_raid  = bool
  })
| 
{
  "disk_bus": "SCSI",
  "disk_count": 1,
  "disk_raid": false,
  "disk_size": 3500,
  "disk_type": "pd-ssd"
}
| no | diff --git a/community/modules/file-system/DDN-EXAScaler/variables.tf b/community/modules/file-system/DDN-EXAScaler/variables.tf index 1cf61be22f..d5c937e008 100644 --- a/community/modules/file-system/DDN-EXAScaler/variables.tf +++ b/community/modules/file-system/DDN-EXAScaler/variables.tf @@ -463,6 +463,6 @@ variable "prefix" { variable "labels" { description = "Labels to add to EXAScaler Cloud deployment. Key-value pairs." - type = map(any) + type = map(string) default = {} } From bbcb6371586770a044c98496e3d2b08b8da2c4f3 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Fri, 17 Mar 2023 16:56:24 +0000 Subject: [PATCH 024/100] Make readers usable outside of package modulereader - Add constructors; - Minor refactoring of tests. --- pkg/modulereader/packerreader.go | 5 +++++ pkg/modulereader/resreader.go | 4 ++-- pkg/modulereader/resreader_test.go | 25 ++++++++++++++----------- pkg/modulereader/tfreader.go | 5 +++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/pkg/modulereader/packerreader.go b/pkg/modulereader/packerreader.go index e61a0f8c41..8f630c41fd 100644 --- a/pkg/modulereader/packerreader.go +++ b/pkg/modulereader/packerreader.go @@ -31,6 +31,11 @@ type PackerReader struct { allModInfo map[string]ModuleInfo } +// NewPackerReader is a constructor for PackerReader +func NewPackerReader() PackerReader { + return PackerReader{allModInfo: map[string]ModuleInfo{}} +} + // SetInfo sets the module info for a module key'd on the source func (r PackerReader) SetInfo(source string, modInfo ModuleInfo) { r.allModInfo[source] = modInfo diff --git a/pkg/modulereader/resreader.go b/pkg/modulereader/resreader.go index 2f3c34d283..08ca1c9da2 100644 --- a/pkg/modulereader/resreader.go +++ b/pkg/modulereader/resreader.go @@ -103,8 +103,8 @@ type ModReader interface { } var kinds = map[string]ModReader{ - "terraform": TFReader{allModInfo: make(map[string]ModuleInfo)}, - "packer": PackerReader{allModInfo: make(map[string]ModuleInfo)}, + "terraform": NewTFReader(), + "packer": NewPackerReader(), } // IsValidKind returns true if the kind input is valid diff --git a/pkg/modulereader/resreader_test.go b/pkg/modulereader/resreader_test.go index dfe52eec31..ad26d26696 100644 --- a/pkg/modulereader/resreader_test.go +++ b/pkg/modulereader/resreader_test.go @@ -193,27 +193,30 @@ func (s *MySuite) TestGetHCLInfo(c *C) { } // tfreader.go -func (s *MySuite) TestGetInfo_TFWriter(c *C) { - reader := TFReader{allModInfo: make(map[string]ModuleInfo)} - moduleInfo, err := reader.GetInfo(terraformDir) +func (s *MySuite) TestGetInfo_TFReder(c *C) { + reader := NewTFReader() + info, err := reader.GetInfo(terraformDir) c.Assert(err, IsNil) - c.Assert(moduleInfo.Inputs[0].Name, Equals, "test_variable") - c.Assert(moduleInfo.Outputs[0].Name, Equals, "test_output") + c.Check(info, DeepEquals, ModuleInfo{ + Inputs: []VarInfo{{Name: "test_variable", Type: "string", Description: "This is just a test", Required: true}}, + Outputs: []VarInfo{{Name: "test_output", Type: "", Description: "This is just a test"}}, + }) + } // packerreader.go func (s *MySuite) TestGetInfo_PackerReader(c *C) { // Didn't already exist, succeeds - reader := PackerReader{allModInfo: make(map[string]ModuleInfo)} - moduleInfo, err := reader.GetInfo(packerDir) + reader := NewPackerReader() + info, err := reader.GetInfo(packerDir) c.Assert(err, IsNil) - c.Assert(moduleInfo.Inputs[0].Name, Equals, "test_variable") + c.Check(info, DeepEquals, ModuleInfo{ + Inputs: []VarInfo{{Name: "test_variable", Type: "string", Description: "This is just a test", Required: true}}}) // Already exists, succeeds - existingModuleInfo, err := reader.GetInfo(packerDir) + infoAgain, err := reader.GetInfo(packerDir) c.Assert(err, IsNil) - c.Assert( - existingModuleInfo.Inputs[0].Name, Equals, moduleInfo.Inputs[0].Name) + c.Check(infoAgain, DeepEquals, info) } // metareader.go diff --git a/pkg/modulereader/tfreader.go b/pkg/modulereader/tfreader.go index 782b2a8a60..6a7751f952 100644 --- a/pkg/modulereader/tfreader.go +++ b/pkg/modulereader/tfreader.go @@ -23,6 +23,11 @@ type TFReader struct { allModInfo map[string]ModuleInfo } +// NewTFReader is a constructor for TFReader +func NewTFReader() TFReader { + return TFReader{allModInfo: map[string]ModuleInfo{}} +} + // SetInfo sets the module info for a module key'd by the source string func (r TFReader) SetInfo(source string, modInfo ModuleInfo) { r.allModInfo[source] = modInfo From a6a650d51e586f3d48e350793a640ef790fcff22 Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Fri, 17 Mar 2023 17:21:17 -0700 Subject: [PATCH 025/100] Fixes slur partition validation message for older terraform versions --- .../compute/schedmd-slurm-gcp-v5-partition/variables.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf index e6732d8ce6..2884022a8e 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf @@ -97,9 +97,7 @@ EOD default = "ANY_SINGLE_ZONE" validation { condition = contains(["ANY", "ANY_SINGLE_ZONE", "BALANCED"], var.zone_target_shape) - error_message = <<-EOD - Allowed values for zone_target_shape are "ANY", "ANY_SINGLE_ZONE", or "BALANCED". - EOD + error_message = "Allowed values for zone_target_shape are \"ANY\", \"ANY_SINGLE_ZONE\", or \"BALANCED\"." } } From 1c578dec85bedd063d165c4aa50ff3b040fee365 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Fri, 17 Mar 2023 22:08:40 -0700 Subject: [PATCH 026/100] Add a test to enforce contracts on modules interfaces (#1041) - Add package `inspect` for tests and utils; - Add test to check all modules input `labels` have type `map(string)` --- pkg/inspect/list.go | 69 ++++++++++++++++++++++ pkg/inspect/modules_test.go | 112 ++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 pkg/inspect/list.go create mode 100644 pkg/inspect/modules_test.go diff --git a/pkg/inspect/list.go b/pkg/inspect/list.go new file mode 100644 index 0000000000..1bb2f9fc73 --- /dev/null +++ b/pkg/inspect/list.go @@ -0,0 +1,69 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inspect + +import ( + "io/fs" + "path/filepath" + "strings" +) + +// SourceAndKind is source and kind +type SourceAndKind struct { + Source string + Kind string +} + +// ListModules in directory +func ListModules(root string, dir string) ([]SourceAndKind, error) { + ret := []SourceAndKind{} + err := filepath.WalkDir(filepath.Join(root, dir), func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() && d.Name() == ".terraform" { + return filepath.SkipDir + } + src, err := filepath.Rel(root, filepath.Dir(path)) + if err != nil { + return err + } + + if !d.IsDir() && filepath.Ext(d.Name()) == ".tf" { + ret = append(ret, SourceAndKind{src, "terraform"}) + return filepath.SkipDir + } + if !d.IsDir() && strings.HasSuffix(d.Name(), ".pkr.hcl") { + ret = append(ret, SourceAndKind{src, "packer"}) + return filepath.SkipDir + } + return nil + }) + return ret, err +} + +// LocalModules returns source and kind for all local modules +func LocalModules() ([]SourceAndKind, error) { + ret := []SourceAndKind{} + + for _, sub := range []string{"modules", "community/modules"} { + mods, err := ListModules("../../", sub) + if err != nil { + return []SourceAndKind{}, err + } + ret = append(ret, mods...) + } + return ret, nil +} diff --git a/pkg/inspect/modules_test.go b/pkg/inspect/modules_test.go new file mode 100644 index 0000000000..38acf03a9c --- /dev/null +++ b/pkg/inspect/modules_test.go @@ -0,0 +1,112 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package inspect_test + +import ( + "hpc-toolkit/pkg/inspect" + "hpc-toolkit/pkg/modulereader" + "log" + "path/filepath" + "testing" + + "golang.org/x/exp/slices" +) + +type varInfo = modulereader.VarInfo +type predicate = func(modInfo) bool + +type modInfo struct { + modulereader.ModuleInfo + inspect.SourceAndKind +} + +func (m *modInfo) Input(name string) (varInfo, bool) { + ind := slices.IndexFunc(m.Inputs, func(v varInfo) bool { return v.Name == name }) + if ind == -1 { + return varInfo{}, false + } + return m.Inputs[ind], true +} + +var allMods []modInfo = nil + +func getModules() []modInfo { + if allMods != nil { + return allMods + } + sks, err := inspect.LocalModules() + if err != nil { + log.Fatal(err) + } + allMods = []modInfo{} + for _, sk := range sks { + info, err := modulereader.GetModuleInfo(filepath.Join("../..", sk.Source), sk.Kind) + if err != nil { + log.Fatal(err) + } + allMods = append(allMods, modInfo{ModuleInfo: info, SourceAndKind: sk}) + } + return allMods +} + +func query(p predicate) []modInfo { + ret := []modInfo{} + for _, mod := range getModules() { + if p(mod) { + ret = append(ret, mod) + } + } + return ret +} + +func all(ps ...predicate) predicate { + return func(mod modInfo) bool { + for _, p := range ps { + if !p(mod) { + return false + } + } + return true + } +} + +func hasInput(name string) predicate { + return func(mod modInfo) bool { + _, ok := mod.Input(name) + return ok + } +} + +// Fails test if slice is empty, returns not empty slice as is. +func notEmpty[E any](l []E, t *testing.T) []E { + if l == nil || len(l) == 0 { + t.Fatal("Did not expect empty list") + } + return l +} + +// Self-test checks that there are modules to inspect +func TestSanity(t *testing.T) { + notEmpty(query(all()), t) +} + +func TestLabelsType(t *testing.T) { + for _, mod := range notEmpty(query(hasInput("labels")), t) { + labels, _ := mod.Input("labels") + if labels.Type != "map(string)" { + t.Errorf("%s.labels has unexpected type %#v", mod.Source, labels.Type) + } + } +} From 1e25b5bbe9c620829e05ca8d29b934e3f4999631 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:01:20 +0000 Subject: [PATCH 027/100] Bump google.golang.org/api from 0.112.0 to 0.114.0 Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.112.0 to 0.114.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.112.0...v0.114.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 754f892196..6cf802d68e 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/zclconf/go-cty v1.13.0 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a - google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v3 v3.0.1 ) @@ -25,7 +25,7 @@ require ( cloud.google.com/go/serviceusage v1.5.0 github.com/go-git/go-billy/v5 v5.4.1 github.com/googleapis/gax-go/v2 v2.7.1 - google.golang.org/api v0.112.0 + google.golang.org/api v0.114.0 ) require ( @@ -77,6 +77,6 @@ require ( golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/grpc v1.53.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.29.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 62ada3e6cd..6b31aa591e 100644 --- a/go.sum +++ b/go.sum @@ -841,8 +841,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.112.0 h1:iDmzvZ4C086R3+en4nSyIf07HlQKMOX1Xx2dmia/+KQ= -google.golang.org/api v0.112.0/go.mod h1:737UfWHNsOq4F3REUTmb+GN9pugkgNLCayLTfoIKpPc= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -954,8 +954,8 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 h1:QQF+HdiI4iocoxUjjpLgvTYDHKm99C/VtTBFnfiCJos= -google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1008,8 +1008,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e3d6b280136ea7c7218151f09215f7424b59abed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:02:08 +0000 Subject: [PATCH 028/100] Bump github.com/go-git/go-git/v5 from 5.6.0 to 5.6.1 Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.6.0 to 5.6.1. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.6.0...v5.6.1) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 8 ++++---- go.sum | 25 +++++++++++++------------ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 754f892196..6a85a46b64 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( cloud.google.com/go/compute v1.18.0 cloud.google.com/go/storage v1.28.1 // indirect - github.com/go-git/go-git/v5 v5.6.0 + github.com/go-git/go-git/v5 v5.6.1 github.com/hashicorp/go-getter v1.7.0 github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl/v2 v2.16.2 @@ -34,8 +34,8 @@ require ( cloud.google.com/go/iam v0.12.0 // indirect cloud.google.com/go/longrunning v0.4.1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect github.com/agext/levenshtein v1.2.2 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/aws/aws-sdk-go v1.44.122 // indirect @@ -69,7 +69,7 @@ require ( github.com/ulikunitz/xz v0.5.10 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.3.0 // indirect + golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index 62ada3e6cd..eaaa714ac7 100644 --- a/go.sum +++ b/go.sum @@ -197,10 +197,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= -github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= +github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agext/levenshtein v1.2.2 h1:0S/Yg6LYmFJ5stwQeRp6EeOcCbj7xiqQSdNelsXvaqE= github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -258,13 +258,12 @@ github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4x github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.6.0 h1:JvBdYfcttd+0kdpuWO7KTu0FYgCf5W0t5VwkWGobaa4= -github.com/go-git/go-git/v5 v5.6.0/go.mod h1:6nmJ0tJ3N4noMV1Omv7rC5FG3/o8Cm51TB4CJp7mRmE= +github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= +github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -500,8 +499,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -591,7 +590,8 @@ golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -705,15 +705,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -725,6 +725,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 373c1e730632987b4c89b1bdc5fe479f78ae9cfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:04:16 +0000 Subject: [PATCH 029/100] Bump lazy-object-proxy from 1.7.1 to 1.9.0 in /community/front-end/ofe Bumps [lazy-object-proxy](https://github.com/ionelmc/python-lazy-object-proxy) from 1.7.1 to 1.9.0. - [Release notes](https://github.com/ionelmc/python-lazy-object-proxy/releases) - [Changelog](https://github.com/ionelmc/python-lazy-object-proxy/blob/master/CHANGELOG.rst) - [Commits](https://github.com/ionelmc/python-lazy-object-proxy/compare/v1.7.1...v1.9.0) --- updated-dependencies: - dependency-name: lazy-object-proxy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 80dac6fc6f..e35613289b 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -35,7 +35,7 @@ h11==0.13.0 httplib2==0.21.0 idna==3.3 isort==5.12.0 -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 libcst==0.4.1 mccabe==0.6.1 mypy-extensions==0.4.3 From 6f7afcbc41e896f51aa1f5b4deb5e6234e141cfa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:04:21 +0000 Subject: [PATCH 030/100] Bump pyjwt from 2.4.0 to 2.6.0 in /community/front-end/ofe Bumps [pyjwt](https://github.com/jpadilla/pyjwt) from 2.4.0 to 2.6.0. - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.4.0...2.6.0) --- updated-dependencies: - dependency-name: pyjwt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 80dac6fc6f..bacda2328a 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -47,7 +47,7 @@ protobuf==4.22.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 -PyJWT==2.4.0 +PyJWT==2.6.0 pylint==2.12.2 pylint-django==2.5.0 pylint-plugin-utils==0.7 From 8dd3ffa9052f65366cb0e5f3e89e2095f7ecfefb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:04:34 +0000 Subject: [PATCH 031/100] Bump pyparsing from 3.0.7 to 3.0.9 in /community/front-end/ofe Bumps [pyparsing](https://github.com/pyparsing/pyparsing) from 3.0.7 to 3.0.9. - [Release notes](https://github.com/pyparsing/pyparsing/releases) - [Changelog](https://github.com/pyparsing/pyparsing/blob/master/CHANGES) - [Commits](https://github.com/pyparsing/pyparsing/compare/pyparsing_3.0.7...pyparsing_3.0.9) --- updated-dependencies: - dependency-name: pyparsing dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 80dac6fc6f..9a225ac124 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -51,7 +51,7 @@ PyJWT==2.4.0 pylint==2.12.2 pylint-django==2.5.0 pylint-plugin-utils==0.7 -pyparsing==3.0.7 +pyparsing==3.0.9 python3-openid==3.2.0 pytz==2021.3 PyYAML==6.0 From 7a9a7bb27fa39465594b16d6372f4fbf6cff4516 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 11:04:39 +0000 Subject: [PATCH 032/100] Bump platformdirs from 2.5.0 to 3.1.1 in /community/front-end/ofe Bumps [platformdirs](https://github.com/platformdirs/platformdirs) from 2.5.0 to 3.1.1. - [Release notes](https://github.com/platformdirs/platformdirs/releases) - [Changelog](https://github.com/platformdirs/platformdirs/blob/main/CHANGES.rst) - [Commits](https://github.com/platformdirs/platformdirs/compare/2.5.0...3.1.1) --- updated-dependencies: - dependency-name: platformdirs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 80dac6fc6f..7428f3b54a 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -40,7 +40,7 @@ libcst==0.4.1 mccabe==0.6.1 mypy-extensions==0.4.3 oauthlib==3.2.2 -platformdirs==2.5.0 +platformdirs==3.1.1 pre-commit==2.17.0 proto-plus==1.22.2 protobuf==4.22.1 From a06bc7eb1048d09d803af74b376e5cf9710c64db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 12:29:23 +0000 Subject: [PATCH 033/100] Bump github.com/googleapis/gax-go/v2 from 2.7.1 to 2.8.0 Bumps [github.com/googleapis/gax-go/v2](https://github.com/googleapis/gax-go) from 2.7.1 to 2.8.0. - [Release notes](https://github.com/googleapis/gax-go/releases) - [Commits](https://github.com/googleapis/gax-go/compare/v2.7.1...v2.8.0) --- updated-dependencies: - dependency-name: github.com/googleapis/gax-go/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6cf802d68e..f2a2aa7be2 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( require ( cloud.google.com/go/serviceusage v1.5.0 github.com/go-git/go-billy/v5 v5.4.1 - github.com/googleapis/gax-go/v2 v2.7.1 + github.com/googleapis/gax-go/v2 v2.8.0 google.golang.org/api v0.114.0 ) diff --git a/go.sum b/go.sum index 6b31aa591e..21494b9ee2 100644 --- a/go.sum +++ b/go.sum @@ -360,8 +360,8 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= From 24b02fb3b94d1014f34a75c69df68a064eb629f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 12:29:29 +0000 Subject: [PATCH 034/100] Bump github.com/zclconf/go-cty from 1.13.0 to 1.13.1 Bumps [github.com/zclconf/go-cty](https://github.com/zclconf/go-cty) from 1.13.0 to 1.13.1. - [Release notes](https://github.com/zclconf/go-cty/releases) - [Changelog](https://github.com/zclconf/go-cty/blob/main/CHANGELOG.md) - [Commits](https://github.com/zclconf/go-cty/compare/v1.13.0...v1.13.1) --- updated-dependencies: - dependency-name: github.com/zclconf/go-cty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6cf802d68e..bb9607ff71 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/spf13/afero v1.9.5 github.com/spf13/cobra v1.6.1 - github.com/zclconf/go-cty v1.13.0 + github.com/zclconf/go-cty v1.13.1 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 6b31aa591e..2ac4d590e8 100644 --- a/go.sum +++ b/go.sum @@ -475,8 +475,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= -github.com/zclconf/go-cty v1.13.0/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= +github.com/zclconf/go-cty v1.13.1 h1:0a6bRwuiSHtAmqCqNOE+c2oHgepv0ctoxU4FUe43kwc= +github.com/zclconf/go-cty v1.13.1/go.mod h1:YKQzy/7pZ7iq2jNFzy5go57xdxdWoLLpaEp4u238AE0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= From e387881c748c208c172c90e534b283db365fde04 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 12:50:04 +0000 Subject: [PATCH 035/100] Bump cloud.google.com/go/serviceusage from 1.5.0 to 1.6.0 Bumps [cloud.google.com/go/serviceusage](https://github.com/googleapis/google-cloud-go) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/documentai/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/dlp/v1.5.0...dlp/v1.6.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/serviceusage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2a2aa7be2..151d145498 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( ) require ( - cloud.google.com/go/serviceusage v1.5.0 + cloud.google.com/go/serviceusage v1.6.0 github.com/go-git/go-billy/v5 v5.4.1 github.com/googleapis/gax-go/v2 v2.8.0 google.golang.org/api v0.114.0 diff --git a/go.sum b/go.sum index 21494b9ee2..299fa7b24d 100644 --- a/go.sum +++ b/go.sum @@ -165,8 +165,8 @@ cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyW cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= -cloud.google.com/go/serviceusage v1.5.0 h1:fl1AGgOx7E2eyBmH5ofDXT9w8xGvEaEnHYyNYGkxaqg= -cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0 h1:rXyq+0+RSIm3HFypctp7WoXxIA563rn206CfMWdqXX4= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= From c3caa285bb3c7ddf26885d93cb7bbdf469f95e3d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 20 Mar 2023 11:24:57 -0500 Subject: [PATCH 036/100] Update builder image - upgrade system-wide pip - add python3-venv --- tools/cloud-build/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/cloud-build/Dockerfile b/tools/cloud-build/Dockerfile index 0a53f26ed3..43223a2f8d 100644 --- a/tools/cloud-build/Dockerfile +++ b/tools/cloud-build/Dockerfile @@ -23,7 +23,7 @@ RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - && \ dnsutils \ shellcheck && \ apt-add-repository "deb [arch=$(dpkg --print-architecture)] https://apt.releases.hashicorp.com bullseye main" && \ - apt-get -y update && apt-get install -y unzip python3-pip terraform packer jq && \ + apt-get -y update && apt-get install -y unzip python3-pip python3-venv terraform packer jq && \ echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \ | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \ @@ -45,8 +45,9 @@ RUN go install github.com/terraform-docs/terraform-docs@latest && \ WORKDIR /ghpc-tmp COPY ./ ./ -RUN pip install -r https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.3.0/scripts/requirements.txt && \ - pip install -r tools/cloud-build/requirements.txt && \ +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.6.0/scripts/requirements.txt && \ + pip install --no-cache-dir -r tools/cloud-build/requirements.txt && \ rm -rf ~/.cache/pip/* RUN make ghpc From 2ef34abfcfa923fa7a19a7fb5204a46e7e3624cc Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Fri, 17 Mar 2023 23:14:22 +0000 Subject: [PATCH 037/100] Minor refactoring of code around variables - Remove unused `ResolveGlobalVariables`; - Compile regexes once and remove exessive error handling; --- pkg/config/config.go | 26 ++++---------------- pkg/config/config_test.go | 51 --------------------------------------- pkg/config/expand.go | 44 ++++++++++++--------------------- 3 files changed, 21 insertions(+), 100 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 546816a321..166c46fd4f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,7 +38,6 @@ import ( const ( expectedVarFormat string = "$(vars.var_name) or $(module_id.output_name)" expectedModFormat string = "$(module_id) or $(group_id.module_id)" - matchLabelExp string = `^[\p{Ll}\p{Lo}\p{N}_-]{1,63}$` ) var errorMessages = map[string]string{ @@ -632,11 +631,7 @@ func (dc *DeploymentConfig) SetBackendConfig(cliBEConfigVars []string) error { // IsLiteralVariable returns true if string matches variable ((ctx.name)) func IsLiteralVariable(str string) bool { - match, err := regexp.MatchString(literalExp, str) - if err != nil { - log.Fatalf("Failed checking if variable is a literal: %v", err) - } - return match + return literalExp.MatchString(str) } // IdentifyLiteralVariable returns @@ -644,8 +639,7 @@ func IsLiteralVariable(str string) bool { // string: variable name (e.g. "project_id") // bool: true/false reflecting success func IdentifyLiteralVariable(str string) (string, string, bool) { - re := regexp.MustCompile(literalSplitExp) - contents := re.FindStringSubmatch(str) + contents := literalSplitExp.FindStringSubmatch(str) if len(contents) != 3 { return "", "", false } @@ -655,8 +649,7 @@ func IdentifyLiteralVariable(str string) (string, string, bool) { // HandleLiteralVariable is exported for use in modulewriter as well func HandleLiteralVariable(str string) string { - re := regexp.MustCompile(literalExp) - contents := re.FindStringSubmatch(str) + contents := literalExp.FindStringSubmatch(str) if len(contents) != 2 { log.Fatalf("Incorrectly formatted literal variable: %s", str) } @@ -742,22 +735,13 @@ func (err *InputValueError) Error() string { return fmt.Sprintf("%v input error, cause: %v", err.inputKey, err.cause) } -// ResolveGlobalVariables will resolve literal variables "((var.*))" in the -// provided map to their corresponding value in the global variables of the -// Blueprint. -func (b Blueprint) ResolveGlobalVariables(ctyVars map[string]cty.Value) error { - origin, err := ConvertMapToCty(b.Vars) - if err != nil { - return fmt.Errorf("error converting deployment variables to cty: %w", err) - } - return ResolveVariables(ctyVars, origin) -} +var matchLabelExp *regexp.Regexp = regexp.MustCompile(`^[\p{Ll}\p{Lo}\p{N}_-]{1,63}$`) // isValidLabelValue checks if a string is a valid value for a GCP label. // For more information on valid label values, see the docs at: // https://cloud.google.com/resource-manager/docs/creating-managing-labels#requirements func isValidLabelValue(value string) bool { - return regexp.MustCompile(matchLabelExp).MatchString(value) + return matchLabelExp.MatchString(value) } // DeploymentName returns the deployment_name from the config and does approperate checks. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index b3c9cbb51f..02747d6c66 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -926,57 +926,6 @@ func (s *MySuite) TestConvertMapToCty(c *C) { c.Assert(found, Equals, false) } -func (s *MySuite) TestResolveGlobalVariables(c *C) { - var err error - var testkey1 = "testkey1" - var testkey2 = "testkey2" - var testkey3 = "testkey3" - dc := getDeploymentConfigForTest() - ctyMap := make(map[string]cty.Value) - err = dc.Config.ResolveGlobalVariables(ctyMap) - c.Assert(err, IsNil) - - // confirm plain string is unchanged and does not error - testCtyString := cty.StringVal("testval") - ctyMap[testkey1] = testCtyString - err = dc.Config.ResolveGlobalVariables(ctyMap) - c.Assert(err, IsNil) - c.Assert(ctyMap[testkey1], Equals, testCtyString) - - // confirm literal, non-global, variable is unchanged and does not error - testCtyString = cty.StringVal("((module.testval))") - ctyMap[testkey1] = testCtyString - err = dc.Config.ResolveGlobalVariables(ctyMap) - c.Assert(err, IsNil) - c.Assert(ctyMap[testkey1], Equals, testCtyString) - - // confirm failed resolution of a literal global - testCtyString = cty.StringVal("((var.test_global_var))") - ctyMap[testkey1] = testCtyString - err = dc.Config.ResolveGlobalVariables(ctyMap) - c.Assert(err, NotNil) - c.Assert(err.Error(), Matches, ".*Unsupported attribute;.*") - - // confirm successful resolution of literal globals in presence of other strings - testGlobalVarString := "test_global_string" - testGlobalValString := "testval" - testGlobalVarBool := "test_global_bool" - testGlobalValBool := "testval" - testPlainString := "plain-string" - dc.Config.Vars[testGlobalVarString] = testGlobalValString - dc.Config.Vars[testGlobalVarBool] = testGlobalValBool - testCtyString = cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarString)) - testCtyBool := cty.StringVal(fmt.Sprintf("((var.%s))", testGlobalVarBool)) - ctyMap[testkey1] = testCtyString - ctyMap[testkey2] = testCtyBool - ctyMap[testkey3] = cty.StringVal(testPlainString) - err = dc.Config.ResolveGlobalVariables(ctyMap) - c.Assert(err, IsNil) - c.Assert(ctyMap[testkey1], Equals, cty.StringVal(testGlobalValString)) - c.Assert(ctyMap[testkey2], Equals, cty.StringVal(testGlobalValBool)) - c.Assert(ctyMap[testkey3], Equals, cty.StringVal(testPlainString)) -} - func (s *MySuite) TestCheckMovedModules(c *C) { dc := DeploymentConfig{ diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 88fad8cdc8..c844394783 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -29,21 +29,24 @@ import ( ) const ( - blueprintLabel string = "ghpc_blueprint" - deploymentLabel string = "ghpc_deployment" - roleLabel string = "ghpc_role" - simpleVariableExp string = `^\$\((.*)\)$` - deploymentVariableExp string = `^\$\(vars\.(.*)\)$` + blueprintLabel string = "ghpc_blueprint" + deploymentLabel string = "ghpc_deployment" + roleLabel string = "ghpc_role" +) + +var ( // Checks if a variable exists only as a substring, ex: // Matches: "a$(vars.example)", "word $(vars.example)", "word$(vars.example)", "$(vars.example)" // Doesn't match: "\$(vars.example)", "no variable in this string" - anyVariableExp string = `(^|[^\\])\$\((.*?)\)` - literalExp string = `^\(\((.*)\)\)$` + anyVariableExp *regexp.Regexp = regexp.MustCompile(`(^|[^\\])\$\((.*?)\)`) + literalExp *regexp.Regexp = regexp.MustCompile(`^\(\((.*)\)\)$`) + simpleVariableExp *regexp.Regexp = regexp.MustCompile(`^\$\((.*)\)$`) // the greediness and non-greediness of expression below is important // consume all whitespace at beginning and end // consume only up to first period to get variable source // consume only up to whitespace to get variable name - literalSplitExp string = `^\(\([[:space:]]*(.*?)\.(.*?)[[:space:]]*\)\)$` + literalSplitExp *regexp.Regexp = regexp.MustCompile(`^\(\([[:space:]]*(.*?)\.(.*?)[[:space:]]*\)\)$`) + deploymentVariableExp *regexp.Regexp = regexp.MustCompile(`^\$\(vars\.(.*)\)$`) ) // expand expands variables and strings in the yaml config. Used directly by @@ -720,10 +723,8 @@ func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]i // Needs DeploymentGroups, variable string, current group, func expandSimpleVariable(context varContext) (string, error) { - // Get variable contents - re := regexp.MustCompile(simpleVariableExp) - contents := re.FindStringSubmatch(context.varString) + contents := simpleVariableExp.FindStringSubmatch(context.varString) if len(contents) != 2 { // Should always be (match, contents) here err := fmt.Errorf("%s %s, failed to extract contents: %v", errorMessages["invalidVar"], context.varString, contents) @@ -781,8 +782,7 @@ func expandSimpleVariable(context varContext) (string, error) { } func expandVariable(context varContext) (string, error) { - re := regexp.MustCompile(anyVariableExp) - matchall := re.FindAllString(context.varString, -1) + matchall := anyVariableExp.FindAllString(context.varString, -1) errHint := "" for _, element := range matchall { // the regex match will include the first matching character @@ -800,29 +800,17 @@ func expandVariable(context varContext) (string, error) { // isDeploymentVariable checks if the entire string is just a single deployment variable func isDeploymentVariable(str string) bool { - matched, err := regexp.MatchString(deploymentVariableExp, str) - if err != nil { - log.Fatalf("isDeploymentVariable(%s): %v", str, err) - } - return matched + return deploymentVariableExp.MatchString(str) } // isSimpleVariable checks if the entire string is just a single variable func isSimpleVariable(str string) bool { - matched, err := regexp.MatchString(simpleVariableExp, str) - if err != nil { - log.Fatalf("isSimpleVariable(%s): %v", str, err) - } - return matched + return simpleVariableExp.MatchString(str) } // hasVariable checks to see if any variable exists in a string func hasVariable(str string) bool { - matched, err := regexp.MatchString(anyVariableExp, str) - if err != nil { - log.Fatalf("hasVariable(%s): %v", str, err) - } - return matched + return anyVariableExp.MatchString(str) } func handleVariable(prim interface{}, context varContext) (interface{}, error) { From 7db5867c8547b06ab0afe23ae2dcb69ac27842a1 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 20 Mar 2023 12:02:50 -0500 Subject: [PATCH 038/100] Add simple sanity test installing the OFE virtual environment --- tools/cloud-build/hpc-toolkit-pr-validation.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/cloud-build/hpc-toolkit-pr-validation.yaml b/tools/cloud-build/hpc-toolkit-pr-validation.yaml index 2c740cb325..aba23c997c 100644 --- a/tools/cloud-build/hpc-toolkit-pr-validation.yaml +++ b/tools/cloud-build/hpc-toolkit-pr-validation.yaml @@ -53,6 +53,20 @@ steps: time addlicense -check -ignore **/lysozyme-example/submit.sh . \ || { echo "addlicense failed"; exit 1; } time make tests +- id: ofe-virtual-env + waitFor: + - git-fetch-unshallow + name: >- + us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/hpc-toolkit-builder + entrypoint: /bin/bash + args: + - '-c' + - | + set -e + python3 -m venv /opt/ofe + source /opt/ofe/bin/activate + pip install --upgrade pip + pip install --dry-run --no-cache-dir -r community/front-end/ofe/requirements.txt timeout: "1200s" options: machineType: N1_HIGHCPU_8 From 2e0eb06e9e959f3a423d6ceae89b309cdb5f3397 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Mon, 20 Mar 2023 20:24:13 +0000 Subject: [PATCH 039/100] Order settings alphabettically in main.tf Introduce a stable order to simplyfy comparison of produced deployments. --- pkg/modulewriter/tfwriter.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/modulewriter/tfwriter.go b/pkg/modulewriter/tfwriter.go index 30f89a45cf..697dff335f 100644 --- a/pkg/modulewriter/tfwriter.go +++ b/pkg/modulewriter/tfwriter.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "github.com/hashicorp/hcl/v2/hclsyntax" @@ -247,7 +248,8 @@ func writeMain( moduleBody.SetAttributeValue("source", moduleSource) // For each Setting - for setting, value := range ctySettings { + for _, setting := range orderSettings(ctySettings) { + value, _ := ctySettings[setting] if setting == "labels" { // Manually compose merge(var.labels, {mod.labels}) using tokens mergeBytes := []byte("merge(var.labels, ") @@ -291,6 +293,7 @@ func writeMain( hclBytes := handleLiteralVariables(hclFile.Bytes()) hclBytes = escapeLiteralVariables(hclBytes) hclBytes = escapeBlueprintVariables(hclBytes) + hclBytes = hclwrite.Format(hclBytes) if err := appendHCLToFile(mainPath, hclBytes); err != nil { return fmt.Errorf("error writing HCL to main.tf file: %v", err) } @@ -469,3 +472,12 @@ func (w TFWriter) restoreState(deploymentDir string) error { } return nil } + +func orderSettings[T any](settings map[string]T) []string { + keys := make([]string, 0, len(settings)) + for k := range settings { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} From 51b2aaaabd76ae31b5142f542b50a1dbcf9f099f Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Mon, 20 Mar 2023 14:56:08 -0700 Subject: [PATCH 040/100] Add links to related material on YouTube --- docs/videos/build-your-own-blueprint/README.md | 4 ++-- docs/videos/healthcare-and-life-sciences/README.md | 3 ++- .../lysozyme-example/README.md | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/videos/build-your-own-blueprint/README.md b/docs/videos/build-your-own-blueprint/README.md index a489abdc7a..6b575423a0 100644 --- a/docs/videos/build-your-own-blueprint/README.md +++ b/docs/videos/build-your-own-blueprint/README.md @@ -1,5 +1,5 @@ # Build Your Own Blueprint Video The [blueprint in this folder](./batch-high-io.yaml) was produced as part of a -training video on how to build your own blueprint from scratch. A link to the -video will be posted here once it has been released. +training video on how to build your own blueprint from scratch. Watch the +YouTube video [here](https://youtu.be/Mb7aOmCdKZU). diff --git a/docs/videos/healthcare-and-life-sciences/README.md b/docs/videos/healthcare-and-life-sciences/README.md index dd35bc4728..78c9a96702 100644 --- a/docs/videos/healthcare-and-life-sciences/README.md +++ b/docs/videos/healthcare-and-life-sciences/README.md @@ -19,7 +19,8 @@ Once deployed, you can test the cluster by running an example workload: cluster is working as expected. - [Lysozyme Example](./lysozyme-example/README.md): This example demonstrates a real life case of simulating the Lysozyme protein in water. It is a multi-step - GPU enabled GROMACS simulation. + GPU enabled GROMACS simulation. This example was featured in + [this YouTube Video](https://youtu.be/kJ-naSow7GQ). ## Architecture diff --git a/docs/videos/healthcare-and-life-sciences/lysozyme-example/README.md b/docs/videos/healthcare-and-life-sciences/lysozyme-example/README.md index 54738f998c..f18373ca18 100644 --- a/docs/videos/healthcare-and-life-sciences/lysozyme-example/README.md +++ b/docs/videos/healthcare-and-life-sciences/lysozyme-example/README.md @@ -1,8 +1,9 @@ # Lysozyme Example -This example demonstrates a real life case of simulating the Lysozyme protein -in water. It uses the HCLS blueprint to run a multi-step GPU enabled GROMACS -run. +This example demonstrates a real life case of simulating the Lysozyme protein in +water. It uses the HCLS blueprint to run a multi-step GPU enabled GROMACS run. +This example was featured in +[this YouTube Video](https://youtu.be/kJ-naSow7GQ). This example has been adapted with changes from tutorials by: From a43800c20fe1e1b0a472993a63ae064af687202a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:12:13 +0000 Subject: [PATCH 041/100] Bump github.com/hashicorp/go-getter from 1.7.0 to 1.7.1 Bumps [github.com/hashicorp/go-getter](https://github.com/hashicorp/go-getter) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/hashicorp/go-getter/releases) - [Changelog](https://github.com/hashicorp/go-getter/blob/main/.goreleaser.yml) - [Commits](https://github.com/hashicorp/go-getter/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-getter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2806cb3c6f..af03cce168 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cloud.google.com/go/compute v1.18.0 cloud.google.com/go/storage v1.28.1 // indirect github.com/go-git/go-git/v5 v5.6.1 - github.com/hashicorp/go-getter v1.7.0 + github.com/hashicorp/go-getter v1.7.1 github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl/v2 v2.16.2 github.com/hashicorp/terraform-config-inspect v0.0.0-20221020162138-81db043ad408 diff --git a/go.sum b/go.sum index f77fcbe80b..9c305a87ef 100644 --- a/go.sum +++ b/go.sum @@ -366,8 +366,8 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.0 h1:bzrYP+qu/gMrL1au7/aDvkoOVGUJpeKBgbqRHACAFDY= -github.com/hashicorp/go-getter v1.7.0/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= +github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= From 22257799305ed6014135715967f0ed620c86f7c2 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 21 Mar 2023 13:28:46 -0500 Subject: [PATCH 042/100] Resolve merge conflict between develop and v1.15.0 release --- .../scheduler/schedmd-slurm-gcp-v5-controller/versions.tf | 2 +- .../modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf index 3304d1f4a9..71fb304d37 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.14.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.15.0" } required_version = ">= 0.14.0" } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf index cd1baf2540..f63f59e9bf 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.14.1" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.15.0" } required_version = ">= 0.14.0" } From f4e0166996d87424ba05ca8e7d799cc73e7a1b6d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 21 Mar 2023 13:36:38 -0500 Subject: [PATCH 043/100] Increase version of Terraform google providers --- pkg/modulewriter/tfversions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/modulewriter/tfversions.go b/pkg/modulewriter/tfversions.go index 979e792da3..ede95b996d 100644 --- a/pkg/modulewriter/tfversions.go +++ b/pkg/modulewriter/tfversions.go @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "~> 4.56.0" + version = "~> 4.58.0" } google-beta = { source = "hashicorp/google-beta" - version = "~> 4.56.0" + version = "~> 4.58.0" } } } From ede75e977db30bb3b32c262b147e7b9a3e4b54fb Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Tue, 21 Mar 2023 16:50:53 +0000 Subject: [PATCH 044/100] Use stable order while writing variables and backend configs --- pkg/modulewriter/tfwriter.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/modulewriter/tfwriter.go b/pkg/modulewriter/tfwriter.go index 697dff335f..746ff7ec56 100644 --- a/pkg/modulewriter/tfwriter.go +++ b/pkg/modulewriter/tfwriter.go @@ -172,7 +172,8 @@ func writeVariables(vars map[string]cty.Value, dst string) error { hclBody := hclFile.Body() // for each variable - for k, v := range vars { + for _, k := range orderKeys(vars) { + v, _ := vars[k] // Create variable block hclBlock := hclBody.AppendNewBlock("variable", []string{k}) blockBody := hclBlock.Body() @@ -218,8 +219,8 @@ func writeMain( tfBody := hclBody.AppendNewBlock("terraform", []string{}).Body() backendBlock := tfBody.AppendNewBlock("backend", []string{tfBackend.Type}) backendBody := backendBlock.Body() - for setting, value := range tfConfig { - backendBody.SetAttributeValue(setting, value) + for _, setting := range orderKeys(tfConfig) { + backendBody.SetAttributeValue(setting, tfConfig[setting]) } hclBody.AppendNewline() } @@ -248,7 +249,7 @@ func writeMain( moduleBody.SetAttributeValue("source", moduleSource) // For each Setting - for _, setting := range orderSettings(ctySettings) { + for _, setting := range orderKeys(ctySettings) { value, _ := ctySettings[setting] if setting == "labels" { // Manually compose merge(var.labels, {mod.labels}) using tokens @@ -473,7 +474,7 @@ func (w TFWriter) restoreState(deploymentDir string) error { return nil } -func orderSettings[T any](settings map[string]T) []string { +func orderKeys[T any](settings map[string]T) []string { keys := make([]string, 0, len(settings)) for k := range settings { keys = append(keys, k) From 48a335b23781ae1038f66efd5812acd01a800ac6 Mon Sep 17 00:00:00 2001 From: Skyler Malinowski Date: Wed, 22 Mar 2023 11:12:20 -0400 Subject: [PATCH 045/100] Update to slurm-gcp 5.6.2 (from 5.6.0) --- .../schedmd-slurm-gcp-v5-node-group/variables.tf | 2 +- .../schedmd-slurm-gcp-v5-partition/README.md | 2 +- .../schedmd-slurm-gcp-v5-partition/main.tf | 2 +- .../schedmd-slurm-gcp-v5-partition/variables.tf | 2 +- .../schedmd-slurm-gcp-v5-controller/README.md | 16 ++++++++-------- .../schedmd-slurm-gcp-v5-controller/main.tf | 4 ++-- .../schedmd-slurm-gcp-v5-controller/variables.tf | 2 +- .../schedmd-slurm-gcp-v5-hybrid/README.md | 14 +++++++------- .../schedmd-slurm-gcp-v5-hybrid/main.tf | 2 +- .../schedmd-slurm-gcp-v5-login/README.md | 14 +++++++------- .../scheduler/schedmd-slurm-gcp-v5-login/main.tf | 4 ++-- .../schedmd-slurm-gcp-v5-login/variables.tf | 2 +- .../demo-with-cloud-controller-instructions.md | 2 +- docs/hybrid-slurm-cluster/deploy-instructions.md | 4 ++-- .../hybrid-slurm-cluster/on-prem-instructions.md | 16 ++++++++-------- tools/cloud-build/Dockerfile | 2 +- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf index 2f7e345c2e..42d365a614 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/variables.tf @@ -15,7 +15,7 @@ */ # Most variables have been sourced and modified from the SchedMD/slurm-gcp -# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 variable "project_id" { description = "Project in which the HPC deployment will be created." diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md b/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md index 5450d73531..847556acfd 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md @@ -157,7 +157,7 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [slurm\_partition](#module\_slurm\_partition) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_partition | 5.6.0 | +| [slurm\_partition](#module\_slurm\_partition) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_partition | 5.6.2 | ## Resources diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf index 55f8e42d85..384610c268 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/main.tf @@ -40,7 +40,7 @@ data "google_compute_zones" "available" { } module "slurm_partition" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_partition?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_partition?ref=5.6.2" slurm_cluster_name = local.slurm_cluster_name partition_nodes = var.node_groups diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf index 2884022a8e..89ef0de009 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf @@ -15,7 +15,7 @@ */ # Most variables have been sourced and modified from the SchedMD/slurm-gcp -# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 variable "deployment_name" { description = "Name of the deployment." diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md index 690194cdd7..5426d24650 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md @@ -17,11 +17,11 @@ controller for optimal performance at different scales. > Python3 (>=3.6.0, <4.0.0) must be installed along with the pip packages listed in the > [requirements.txt] file of [SchedMD/slurm-gcp]. -[SchedMD/slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 -[slurm\_controller\_instance]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster/modules/slurm_controller_instance -[slurm\_instance\_template]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster/modules/slurm_instance_template +[SchedMD/slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 +[slurm\_controller\_instance]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_controller_instance +[slurm\_instance\_template]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_instance_template [slurm-ug]: https://goo.gle/slurm-gcp-user-guide. -[requirements.txt]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/scripts/requirements.txt +[requirements.txt]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/scripts/requirements.txt [enable\_cleanup\_compute]: #input\_enable\_cleanup\_compute [enable\_cleanup\_subscriptions]: #input\_enable\_cleanup\_subscriptions [enable\_reconfigure]: #input\_enable\_reconfigure @@ -74,7 +74,7 @@ This option has some additional requirements: development environment deploying the cluster. One can use following commands: ```bash - wget https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.6.0/scripts/requirements.txt + wget https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.6.2/scripts/requirements.txt pip3 install -r requirements.txt --user ``` @@ -84,7 +84,7 @@ This option has some additional requirements: default config project, run the following command: `gcloud config set core/<>` -[optdeps]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster#optional +[optdeps]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster#optional ## Custom Images @@ -148,8 +148,8 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [slurm\_controller\_instance](#module\_slurm\_controller\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_instance | 5.6.0 | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 5.6.0 | +| [slurm\_controller\_instance](#module\_slurm\_controller\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_instance | 5.6.2 | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 5.6.2 | ## Resources diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/main.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/main.tf index ac8fb1a679..5f6603fb51 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/main.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/main.tf @@ -56,7 +56,7 @@ data "google_compute_default_service_account" "default" { } module "slurm_controller_instance" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_instance?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_instance?ref=5.6.2" access_config = local.access_config slurm_cluster_name = local.slurm_cluster_name @@ -92,7 +92,7 @@ module "slurm_controller_instance" { } module "slurm_controller_template" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=5.6.2" additional_disks = local.additional_disks can_ip_forward = var.can_ip_forward diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf index 138702a5b3..6651b68e4b 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf @@ -15,7 +15,7 @@ */ # Most variables have been sourced and modified from the SchedMD/slurm-gcp -# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 variable "access_config" { description = "Access configurations, i.e. IPs via which the VM instance can be accessed via the Internet." diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/README.md index 79cbd95622..feff9fb973 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/README.md @@ -38,7 +38,7 @@ manually. This will require addition configuration and verification of permissions. For more information see the [hybrid.md] documentation on [slurm-gcp]. -[slurm-controller-hybrid]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster/modules/slurm_controller_hybrid +[slurm-controller-hybrid]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_controller_hybrid > **_NOTE:_** The hybrid module requires the following dependencies to be > installed on the system deploying the module: @@ -58,15 +58,15 @@ permissions. For more information see the [hybrid.md] documentation on [pyyaml]: https://pypi.org/project/PyYAML/ [google-api-python-client]: https://pypi.org/project/google-api-python-client/ [google-cloud-pubsub]: https://pypi.org/project/google-cloud-pubsub/ -[requirements.txt]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/scripts/requirements.txt +[requirements.txt]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/scripts/requirements.txt ### Manual Configuration This module *does not* complete the installation of hybrid partitions on your slurm cluster. After deploying, you must follow the steps listed out in the [hybrid.md] documentation under [manual steps]. -[hybrid.md]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/docs/hybrid.md -[manual steps]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/docs/hybrid.md#manual-configurations +[hybrid.md]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/docs/hybrid.md +[manual steps]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/docs/hybrid.md#manual-configurations ### Example Usage The hybrid module can be added to a blueprint as follows: @@ -146,10 +146,10 @@ strongly advise only using versions 21 or 22 when using this module. Attempting to use this module with any version older than 21 may lead to unexpected results. -[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 [pre-existing-network-storage]: ../../../../modules/file-system/pre-existing-network-storage/ [schedmd-slurm-gcp-v5-partition]: ../../compute/schedmd-slurm-gcp-v5-partition/ -[packer templates]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/packer +[packer templates]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/packer ## License @@ -181,7 +181,7 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [slurm\_controller\_instance](#module\_slurm\_controller\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_hybrid | 5.6.0 | +| [slurm\_controller\_instance](#module\_slurm\_controller\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_hybrid | 5.6.2 | ## Resources diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/main.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/main.tf index 92f17a87df..0088ff0726 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/main.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/main.tf @@ -28,7 +28,7 @@ locals { } module "slurm_controller_instance" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_hybrid?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_controller_hybrid?ref=5.6.2" project_id = var.project_id slurm_cluster_name = local.slurm_cluster_name diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/README.md index 4f5ed15280..aa137865e9 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/README.md @@ -5,9 +5,9 @@ This module creates a login node for a Slurm cluster based on the terraform modules. The login node is used in conjunction with the [Slurm controller](../schedmd-slurm-gcp-v5-controller/README.md). -[SchedMD/slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 -[slurm\_login\_instance]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster/modules/slurm_login_instance -[slurm\_instance\_template]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/terraform/slurm_cluster/modules/slurm_instance_template +[SchedMD/slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 +[slurm\_login\_instance]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_login_instance +[slurm\_instance\_template]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_instance_template ### Example @@ -49,8 +49,8 @@ The HPC Toolkit team maintains the wrapper around the [slurm-on-gcp] terraform modules. For support with the underlying modules, see the instructions in the [slurm-gcp README][slurm-gcp-readme]. -[slurm-on-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 -[slurm-gcp-readme]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0#slurm-on-google-cloud-platform +[slurm-on-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 +[slurm-gcp-readme]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2#slurm-on-google-cloud-platform ## License @@ -85,8 +85,8 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [slurm\_login\_instance](#module\_slurm\_login\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_login_instance | 5.6.0 | -| [slurm\_login\_template](#module\_slurm\_login\_template) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 5.6.0 | +| [slurm\_login\_instance](#module\_slurm\_login\_instance) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_login_instance | 5.6.2 | +| [slurm\_login\_template](#module\_slurm\_login\_template) | github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 5.6.2 | ## Resources diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/main.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/main.tf index b0e2bd9497..10759298a0 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/main.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/main.tf @@ -52,7 +52,7 @@ data "google_compute_default_service_account" "default" { } module "slurm_login_template" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=5.6.2" additional_disks = local.additional_disks can_ip_forward = var.can_ip_forward @@ -91,7 +91,7 @@ module "slurm_login_template" { } module "slurm_login_instance" { - source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_login_instance?ref=5.6.0" + source = "github.com/SchedMD/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_login_instance?ref=5.6.2" access_config = local.access_config slurm_cluster_name = local.slurm_cluster_name diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/variables.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/variables.tf index 0fb87f0fa2..3ab8dad7f8 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/variables.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/variables.tf @@ -15,7 +15,7 @@ */ # Most variables have been sourced and modified from the SchedMD/slurm-gcp -# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +# github repository: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 variable "project_id" { type = string diff --git a/docs/hybrid-slurm-cluster/demo-with-cloud-controller-instructions.md b/docs/hybrid-slurm-cluster/demo-with-cloud-controller-instructions.md index a1d675e1d7..11ae11d835 100644 --- a/docs/hybrid-slurm-cluster/demo-with-cloud-controller-instructions.md +++ b/docs/hybrid-slurm-cluster/demo-with-cloud-controller-instructions.md @@ -22,7 +22,7 @@ for use with an on-premise slurm-cluster. > further testing is done, documentation on applying the hybrid module to > on-premise slurm clusters will be added and expanded. -[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 ## Definitions diff --git a/docs/hybrid-slurm-cluster/deploy-instructions.md b/docs/hybrid-slurm-cluster/deploy-instructions.md index a76ce9fb64..5b40acc3f6 100644 --- a/docs/hybrid-slurm-cluster/deploy-instructions.md +++ b/docs/hybrid-slurm-cluster/deploy-instructions.md @@ -260,8 +260,8 @@ sudo systemctl restart slurmctld If the restart did not succeed, the logs at `/var/log/slurm/slurmctld.log` should point you in the right direction. -[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 -[slurm-gcp-hybrid]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/docs/hybrid.md +[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 +[slurm-gcp-hybrid]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/docs/hybrid.md [demo-with-cloud-controller-instructions.md]: ./demo-with-cloud-controller-instructions.md ## Validate the Hybrid Cluster diff --git a/docs/hybrid-slurm-cluster/on-prem-instructions.md b/docs/hybrid-slurm-cluster/on-prem-instructions.md index 8df978065a..b2f2374441 100644 --- a/docs/hybrid-slurm-cluster/on-prem-instructions.md +++ b/docs/hybrid-slurm-cluster/on-prem-instructions.md @@ -39,9 +39,9 @@ detail, as well as how to customize many of these assumptions to fit your needs. deployments in their [hybrid.md] documentation. [hybridmodule]: ../../community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/README.md -[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0 +[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 [slurm\_controller\_hybrid]: https://github.com/SchedMD/slurm-gcp/tree/master/terraform/slurm_cluster/modules/slurm_controller_hybrid -[hybrid.md]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/docs/hybrid.md +[hybrid.md]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/docs/hybrid.md ### NFS Mounts @@ -235,12 +235,12 @@ image created with slurm 21.08.8: partition_name: compute ``` -[slurmgcppacker]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/packer -[example.pkrvars.hcl]: https://github.com/SchedMD/slurm-gcp/tree/5.6.0/packer/example.pkrvars.hcl -[slurmversion]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/packer/variables.pkr.hcl#L97 -[`service_account_scopes`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/packer/variables.pkr.hcl#L166 -[`munge_user`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/ansible/roles/munge/defaults/main.yml#L17 -[`slurm_user`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.0/ansible/roles/slurm/defaults/main.yml#L31 +[slurmgcppacker]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/packer +[example.pkrvars.hcl]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/packer/example.pkrvars.hcl +[slurmversion]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/packer/variables.pkr.hcl#L97 +[`service_account_scopes`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/packer/variables.pkr.hcl#L166 +[`munge_user`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/ansible/roles/munge/defaults/main.yml#L17 +[`slurm_user`]: https://github.com/SchedMD/slurm-gcp/blob/5.6.2/ansible/roles/slurm/defaults/main.yml#L31 ## On Premise Setup diff --git a/tools/cloud-build/Dockerfile b/tools/cloud-build/Dockerfile index 43223a2f8d..efae91c609 100644 --- a/tools/cloud-build/Dockerfile +++ b/tools/cloud-build/Dockerfile @@ -46,7 +46,7 @@ WORKDIR /ghpc-tmp COPY ./ ./ RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -r https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.6.0/scripts/requirements.txt && \ + pip install --no-cache-dir -r https://raw.githubusercontent.com/SchedMD/slurm-gcp/5.6.2/scripts/requirements.txt && \ pip install --no-cache-dir -r tools/cloud-build/requirements.txt && \ rm -rf ~/.cache/pip/* From 734aac1da559581a852f37fa6ce280e80bde81f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:08:32 +0000 Subject: [PATCH 046/100] Bump google-api-python-client in /community/front-end/ofe Bumps [google-api-python-client](https://github.com/googleapis/google-api-python-client) from 2.37.0 to 2.82.0. - [Release notes](https://github.com/googleapis/google-api-python-client/releases) - [Changelog](https://github.com/googleapis/google-api-python-client/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-api-python-client/compare/v2.37.0...v2.82.0) --- updated-dependencies: - dependency-name: google-api-python-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 80dac6fc6f..441fc2d663 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -17,7 +17,7 @@ django-extensions==3.2.1 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 djangorestframework==3.13.1 google-api-core==2.5.0 -google-api-python-client==2.37.0 +google-api-python-client==2.82.0 google-auth==2.6.0 google-auth-httplib2==0.1.0 google-cloud-billing==1.4.1 From 4e66f3a3ede5609a7ce23eff62f5159a9ad0e193 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 21 Mar 2023 15:52:25 -0500 Subject: [PATCH 047/100] Implement exponential backoff in startup-script - Address issue where service account may not have IAM bindings propagated to it (GCP replication process) even though they are created in-order in Terraform. - Avoid false "about to retry" log entry on last iteration of loop --- .../scripts/startup-script/files/get_from_bucket.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/scripts/startup-script/files/get_from_bucket.sh b/modules/scripts/startup-script/files/get_from_bucket.sh index d5d70e5a78..ff47d6e5d5 100644 --- a/modules/scripts/startup-script/files/get_from_bucket.sh +++ b/modules/scripts/startup-script/files/get_from_bucket.sh @@ -48,13 +48,18 @@ stdlib::get_from_bucket() { stdlib::debug "Computed filename='${fname}' given URL." fi [[ -d ${dir} ]] || mkdir "${dir}" - local attempt=1 - local max_attempts=10 - while [[ $attempt -lt $max_attempts ]]; do + local attempt=0 + local max_retries=7 + while [[ $attempt -le $max_retries ]]; do + if [[ $attempt -gt 0 ]]; then + local wait=$((2 ** attempt)) + stdlib::error "Retry attempt ${attempt} of ${max_retries} with exponential backoff: ${wait} seconds." + sleep $wait + fi if stdlib::cmd gsutil -q cp "${url}" "${dir}/${fname}"; then break else - stdlib::error "gsutil reported non-zero exit code fetching ${url}. Retrying (${attempt}/${max_attempts})" + stdlib::error "gsutil reported non-zero exit code fetching ${url}." ((attempt++)) fi done From 552f737c3d46e2ff8b185f6f3bab1161554a2e08 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Wed, 22 Mar 2023 10:59:10 -0500 Subject: [PATCH 048/100] Update Slurm images through tests and examples --- community/examples/AMD/hpc-cluster-amd-slurmv5.yaml | 4 ++-- examples/image-builder.yaml | 2 +- tools/validate_configs/test_configs/node-groups.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml b/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml index a488d53bf3..dac94a89c9 100644 --- a/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml +++ b/community/examples/AMD/hpc-cluster-amd-slurmv5.yaml @@ -70,7 +70,7 @@ deployment_groups: packages: slurm: externals: - - spec: slurm@22-05-3 + - spec: slurm@22-05-8 prefix: /usr/local buildable: False - type: file @@ -169,7 +169,7 @@ deployment_groups: # these images must match the images used by Slurm modules below because # we are building OpenMPI with PMI support in libaries contained in # Slurm installation - family: schedmd-v5-slurm-22-05-6-hpc-centos-7 + family: schedmd-v5-slurm-22-05-8-hpc-centos-7 project: schedmd-slurm-public - id: low_cost_node_group diff --git a/examples/image-builder.yaml b/examples/image-builder.yaml index 3181a447ac..9dd46dbba1 100644 --- a/examples/image-builder.yaml +++ b/examples/image-builder.yaml @@ -53,7 +53,7 @@ deployment_groups: settings: source_image_project_id: [schedmd-slurm-public] # see latest in https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems - source_image_family: schedmd-v5-slurm-22-05-6-hpc-centos-7 + source_image_family: schedmd-v5-slurm-22-05-8-hpc-centos-7 # You can find size of source image by using following command # gcloud compute images describe-from-family --project schedmd-slurm-public disk_size: $(vars.disk_size) diff --git a/tools/validate_configs/test_configs/node-groups.yaml b/tools/validate_configs/test_configs/node-groups.yaml index 1d553cb8f0..d4dad7182f 100644 --- a/tools/validate_configs/test_configs/node-groups.yaml +++ b/tools/validate_configs/test_configs/node-groups.yaml @@ -73,7 +73,7 @@ deployment_groups: name: c60 machine_type: c2-standard-60 instance_image: - name: schedmd-v5-slurm-22-05-4-hpc-centos-7-1665675565 + name: schedmd-v5-slurm-22-05-8-centos-7-1678978029 project: projects/schedmd-slurm-public/global/images - id: node_group3 @@ -82,7 +82,7 @@ deployment_groups: name: cd112 machine_type: c2d-standard-112 instance_image: - family: schedmd-v5-slurm-22-05-6-hpc-centos-7 + family: schedmd-v5-slurm-22-05-8-hpc-centos-7 project: projects/schedmd-slurm-public/global/images/family enable_smt: true From ea2a6854523f622eeb242309290c2645638d9e79 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Wed, 22 Mar 2023 13:54:57 -0500 Subject: [PATCH 049/100] Add deployment_name to HTCondor example VMs Helpful in general, but will also enable HTCondor integration tests to run in parallel. --- community/examples/htcondor-pool.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/community/examples/htcondor-pool.yaml b/community/examples/htcondor-pool.yaml index fbad228998..5a02e53ac4 100644 --- a/community/examples/htcondor-pool.yaml +++ b/community/examples/htcondor-pool.yaml @@ -56,7 +56,8 @@ deployment_groups: - network1 - htcondor_startup_central_manager settings: - name_prefix: central-manager + name_prefix: cm + add_deployment_name_before_prefix: true machine_type: c2-standard-4 disable_public_ips: true service_account: @@ -140,7 +141,8 @@ deployment_groups: - network1 - htcondor_startup_access_point settings: - name_prefix: access-point + name_prefix: ap + add_deployment_name_before_prefix: true machine_type: c2-standard-4 service_account: email: $(htcondor_configure.access_point_service_account) From 4175bdaf3a4d2822ad88d997801569e6040f6214 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Thu, 23 Mar 2023 11:02:32 -0700 Subject: [PATCH 050/100] Remove labels-specific logic from tfwriter. (#1064) Combine labels during `DeploymentConfig.combineLabels`, use `WrapSettingsWith` to express merged labels. ``` $ make && ./ghpc expand tools/validate_configs/test_configs/use-resources.yaml --vars="project_id=X" $ diff --context=3 expanded_orig.yaml expanded.yaml @mr0re1: Rest of listing is similiar *************** *** 171,184 **** - slurm_controller - network1 wrapsettingswith: ! network_storage: ! - flatten( - ) settings: controller_name: ((module.slurm_controller.controller_name)) deployment_name: ((var.deployment_name)) labels: ! ghpc_role: scheduler login_machine_type: n2-standard-4 network_storage: - ((module.homefs.network_storage)) --- 191,208 ---- - slurm_controller - network1 wrapsettingswith: ! labels: ! - merge( - ) + network_storage: + - flatten([ + - '])' settings: controller_name: ((module.slurm_controller.controller_name)) deployment_name: ((var.deployment_name)) labels: ! - ((var.labels)) ! - ghpc_role: scheduler login_machine_type: n2-standard-4 network_storage: - ((module.homefs.network_storage)) $ make && ./ghpc create tools/validate_configs/test_configs/use-resources.yaml --vars="project_id=X" $ diff hpc-slurm-use-modules_orig/primary/main.tf hpc-slurm-use-modules/primary/main.tf @mr0re1 no diff $ make && ./ghpc create examples/image-builder.yaml --vars="project_id=X" $ cat pkg/config/expand_test.go ... labels = { ghpc_blueprint = "image-builder" ghpc_deployment = "image-builder-001" ghpc_role = "packer" } ... ``` --- pkg/config/expand.go | 90 +++++++++++++------- pkg/config/expand_test.go | 118 +++++++++++++++++++------- pkg/modulewriter/modulewriter_test.go | 14 --- pkg/modulewriter/tfwriter.go | 83 +++++++++++------- 4 files changed, 196 insertions(+), 109 deletions(-) diff --git a/pkg/config/expand.go b/pkg/config/expand.go index c844394783..f338fa01a6 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -173,7 +173,7 @@ func (mod *Module) addListValue(settingName string, value string) error { if !found { mod.Settings[settingName] = []interface{}{} mod.createWrapSettingsWith() - mod.WrapSettingsWith[settingName] = []string{"flatten(", ")"} + mod.WrapSettingsWith[settingName] = []string{"flatten([", "])"} } currentValue, ok := mod.Settings[settingName].([]interface{}) if ok { @@ -380,47 +380,73 @@ func (dc *DeploymentConfig) combineLabels() error { } // Add both default labels if they don't already exist - if _, exists := globalLabels[blueprintLabel]; !exists { - globalLabels[blueprintLabel] = defaultLabels[blueprintLabel] - } - if _, exists := globalLabels[deploymentLabel]; !exists { - globalLabels[deploymentLabel] = defaultLabels[deploymentLabel] - } + mergeInLabels(globalLabels, defaultLabels) for iGrp, grp := range dc.Config.DeploymentGroups { - for iMod, mod := range grp.Modules { - // Check if labels are set for this module - if !dc.moduleHasInput(grp.Name, mod.Source, labels) { - continue + for iMod := range grp.Modules { + if err := combineModuleLabels(dc, iGrp, iMod); err != nil { + return err } + } + } + dc.Config.Vars[labels] = globalLabels + return nil +} - var modLabels map[string]interface{} - var ok bool - // If labels aren't already set, prefill them with globals - if _, exists := mod.Settings[labels]; !exists { - modLabels = make(map[string]interface{}) - } else { - // Cast into map so we can index into them - modLabels, ok = mod.Settings[labels].(map[string]interface{}) - - if !ok { - return fmt.Errorf("%s, Module %s, labels type: %T", - errorMessages["settingsLabelType"], mod.ID, mod.Settings[labels]) - } - } +func combineModuleLabels(dc *DeploymentConfig, iGrp int, iMod int) error { + grp := &dc.Config.DeploymentGroups[iGrp] + mod := &grp.Modules[iMod] + mod.createWrapSettingsWith() + labels := "labels" - // Add the role (e.g. compute, network, etc) - if _, exists := modLabels[roleLabel]; !exists { - modLabels[roleLabel] = getRole(mod.Source) - } - dc.Config.DeploymentGroups[iGrp].Modules[iMod].Settings[labels] = - modLabels + // previously expanded blueprint, user written BPs do not use `WrapSettingsWith` + if _, ok := mod.WrapSettingsWith[labels]; ok { + return nil // Do nothing + } + + // Check if labels are set for this module + if !dc.moduleHasInput(grp.Name, mod.Source, labels) { + return nil + } + + var modLabels map[string]interface{} + var err error + + if _, exists := mod.Settings[labels]; !exists { + modLabels = map[string]interface{}{} + } else { + // Cast into map so we can index into them + modLabels, err = toStringInterfaceMap(mod.Settings[labels]) + if err != nil { + return fmt.Errorf("%s, Module %s, labels type: %T", + errorMessages["settingsLabelType"], mod.ID, mod.Settings[labels]) } } - dc.Config.Vars[labels] = globalLabels + // Add the role (e.g. compute, network, etc) + if _, exists := modLabels[roleLabel]; !exists { + modLabels[roleLabel] = getRole(mod.Source) + } + + if mod.Kind == "terraform" { + // Terraform module labels to be expressed as + // `merge(var.labels, { ghpc_role=..., **settings.labels })` + mod.WrapSettingsWith[labels] = []string{"merge(", ")"} + mod.Settings[labels] = []interface{}{"((var.labels))", modLabels} + } else if mod.Kind == "packer" { + mergeInLabels(modLabels, dc.Config.Vars[labels].(map[string]interface{})) + mod.Settings[labels] = modLabels + } return nil } +func mergeInLabels[V interface{}](to map[string]V, from map[string]V) { + for k, v := range from { + if _, exists := to[k]; !exists { + to[k] = v + } + } +} + func applyGlobalVarsInGroup( deploymentGroup DeploymentGroup, modInfo map[string]modulereader.ModuleInfo, diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 57b7ce1d5d..94f83ab86f 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -328,45 +328,99 @@ func (s *MySuite) TestUpdateVariableType(c *C) { } func (s *MySuite) TestCombineLabels(c *C) { - dc := getDeploymentConfigForTest() - - err := dc.combineLabels() - c.Assert(err, IsNil) + infoWithLabels := modulereader.ModuleInfo{Inputs: []modulereader.VarInfo{{Name: "labels"}}} + + dc := DeploymentConfig{ + Config: Blueprint{ + BlueprintName: "simple", + Vars: map[string]interface{}{ + "deployment_name": "golden"}, + DeploymentGroups: []DeploymentGroup{ + { + Name: "lime", + Modules: []Module{ + {Source: "blue/salmon", Kind: "terraform", ID: "coral", Settings: map[string]interface{}{ + "labels": map[string]interface{}{ + "magenta": "orchid", + "ghpc_role": "maroon", + }, + }}, + {Source: "brown/oak", Kind: "terraform", ID: "khaki", Settings: map[string]interface{}{ + // has no labels set + }}, + {Source: "ivory/black", Kind: "terraform", ID: "silver", Settings: map[string]interface{}{ + // has no labels set, also module has no labels input + }}, + }, + }, + { + Name: "pink", + Modules: []Module{ + {Source: "red/velvet", Kind: "packer", ID: "orange", Settings: map[string]interface{}{ + "labels": map[string]interface{}{ + "olive": "teal", + "ghpc_deployment": "navy", + }, + }}, + }, + }, + }, + }, + ModulesInfo: map[string]map[string]modulereader.ModuleInfo{ + "lime": { + "blue/salmon": infoWithLabels, + "brown/oak": infoWithLabels, + "ivory/black": modulereader.ModuleInfo{Inputs: []modulereader.VarInfo{}}, + }, + "pink": { + "red/velvet": infoWithLabels, + }, + }, + } + c.Check(dc.combineLabels(), IsNil) // Were global labels created? - _, exists := dc.Config.Vars["labels"] - c.Assert(exists, Equals, true) - - // Was the ghpc_blueprint label set correctly? - globalLabels := dc.Config.Vars["labels"].(map[string]interface{}) - ghpcBlueprint, exists := globalLabels[blueprintLabel] - c.Assert(exists, Equals, true) - c.Assert(ghpcBlueprint, Equals, dc.Config.BlueprintName) - - // Was the ghpc_deployment label set correctly? - ghpcDeployment, exists := globalLabels[deploymentLabel] - c.Assert(exists, Equals, true) - c.Assert(ghpcDeployment, Equals, "deployment_name") - - // Was "labels" created for the module with no settings? - _, exists = dc.Config.DeploymentGroups[0].Modules[0].Settings["labels"] - c.Assert(exists, Equals, true) - - moduleLabels := dc.Config.DeploymentGroups[0].Modules[0]. - Settings["labels"].(map[string]interface{}) - - // Was the role created correctly? - ghpcRole, exists := moduleLabels[roleLabel] - c.Assert(exists, Equals, true) - c.Assert(ghpcRole, Equals, "other") + c.Check(dc.Config.Vars["labels"], DeepEquals, map[string]interface{}{ + "ghpc_blueprint": "simple", + "ghpc_deployment": "golden", + }) + + lime := dc.Config.DeploymentGroups[0] + // Labels are set and override role + coral := lime.Modules[0] + c.Check(coral.WrapSettingsWith["labels"], DeepEquals, []string{"merge(", ")"}) + c.Check(coral.Settings["labels"], DeepEquals, []interface{}{ + "((var.labels))", + map[string]interface{}{"magenta": "orchid", "ghpc_role": "maroon"}, + }) + // Labels are not set, infer role from module.source + khaki := lime.Modules[1] + c.Check(khaki.WrapSettingsWith["labels"], DeepEquals, []string{"merge(", ")"}) + c.Check(khaki.Settings["labels"], DeepEquals, []interface{}{ + "((var.labels))", + map[string]interface{}{"ghpc_role": "brown"}, + }) + // No labels input + silver := lime.Modules[2] + c.Check(silver.WrapSettingsWith["labels"], IsNil) + c.Check(silver.Settings["labels"], IsNil) + + // Packer, include global include explicitly + // Keep overriden ghpc_deployment=navy + orange := dc.Config.DeploymentGroups[1].Modules[0] + c.Check(orange.WrapSettingsWith["labels"], IsNil) + c.Check(orange.Settings["labels"], DeepEquals, map[string]interface{}{ + "ghpc_blueprint": "simple", + "ghpc_deployment": "navy", + "ghpc_role": "red", + "olive": "teal", + }) // Test invalid labels dc.Config.Vars["labels"] = "notAMap" - err = dc.combineLabels() expectedErrorStr := fmt.Sprintf("%s: found %T", errorMessages["globalLabelType"], dc.Config.Vars["labels"]) - c.Assert(err, ErrorMatches, expectedErrorStr) - + c.Check(dc.combineLabels(), ErrorMatches, expectedErrorStr) } func (s *MySuite) TestApplyGlobalVariables(c *C) { diff --git a/pkg/modulewriter/modulewriter_test.go b/pkg/modulewriter/modulewriter_test.go index a95f9884d6..ec891957ca 100644 --- a/pkg/modulewriter/modulewriter_test.go +++ b/pkg/modulewriter/modulewriter_test.go @@ -498,20 +498,6 @@ func (s *MySuite) TestWriteMain(c *C) { c.Assert(err, IsNil) c.Assert(exists, Equals, true) - // Test with labels setting - testModule.Settings["labels"] = map[string]interface{}{ - "ghpc_role": "testModule", - "custom_label": "", - } - err = writeMain(testModules, testBackend, testMainDir) - c.Assert(err, IsNil) - exists, err = stringExistsInFile("custom_label", mainFilePath) - c.Assert(err, IsNil) - c.Assert(exists, Equals, true) - exists, err = stringExistsInFile("var.labels", mainFilePath) - c.Assert(err, IsNil) - c.Assert(exists, Equals, true) - // Test with Backend testBackend.Type = "gcs" testBackend.Configuration = map[string]interface{}{ diff --git a/pkg/modulewriter/tfwriter.go b/pkg/modulewriter/tfwriter.go index 746ff7ec56..72fcb55c8a 100644 --- a/pkg/modulewriter/tfwriter.go +++ b/pkg/modulewriter/tfwriter.go @@ -251,38 +251,17 @@ func writeMain( // For each Setting for _, setting := range orderKeys(ctySettings) { value, _ := ctySettings[setting] - if setting == "labels" { - // Manually compose merge(var.labels, {mod.labels}) using tokens - mergeBytes := []byte("merge(var.labels, ") - - labelsStr := flattenHCLLabelsMap( - string(hclwrite.TokensForValue(value).Bytes())) - - mergeBytes = append(mergeBytes, []byte(labelsStr)...) - mergeBytes = append(mergeBytes, byte(')')) - - mergeTok := simpleTokenFromString(string(mergeBytes)) - labelsTokens := []*hclwrite.Token{&mergeTok} - - moduleBody.SetAttributeRaw(setting, labelsTokens) - continue - } - if wrap, ok := mod.WrapSettingsWith[setting]; ok { if len(wrap) != 2 { return fmt.Errorf( "invalid length of WrapSettingsWith for %s.%s, expected 2 got %d", mod.ID, setting, len(wrap)) } - wrapBytes := []byte(wrap[0]) - endBytes := []byte(wrap[1]) - - valueStr := hclwrite.TokensForValue(value).Bytes() - wrapBytes = append(wrapBytes, valueStr...) - wrapBytes = append(wrapBytes, endBytes...) - wrapToken := simpleTokenFromString(string(wrapBytes)) - wrapTokens := []*hclwrite.Token{&wrapToken} - moduleBody.SetAttributeRaw(setting, wrapTokens) + toks, err := tokensForWrapped(wrap[0], value, wrap[1]) + if err != nil { + return fmt.Errorf("failed to process %s.%s: %v", mod.ID, setting, err) + } + moduleBody.SetAttributeRaw(setting, toks) } else { // Add attributes moduleBody.SetAttributeValue(setting, value) @@ -301,11 +280,45 @@ func writeMain( return nil } -func flattenHCLLabelsMap(hclString string) string { - hclString = strings.ReplaceAll(hclString, "\"\n", "\",") - hclString = strings.ReplaceAll(hclString, "\n", "") - hclString = strings.Join(strings.Fields(hclString), " ") - return hclString +func tokensForWrapped(pref string, val cty.Value, suf string) (hclwrite.Tokens, error) { + var toks hclwrite.Tokens + if !val.Type().IsListType() && !val.Type().IsTupleType() { + return toks, fmt.Errorf( + "invalid value for wrapped setting, expected sequence, got %#v", val.Type()) + } + prefTok := simpleTokenFromString(pref) + toks = append(toks, &prefTok) + + it, first := val.ElementIterator(), true + for it.Next() { + if !first { + toks = append(toks, &hclwrite.Token{ + Type: hclsyntax.TokenComma, + Bytes: []byte{','}}) + } + _, el := it.Element() + toks = append(toks, tokensForValue(el)...) + first = false + } + + sufTok := simpleTokenFromString(suf) + toks = append(toks, &sufTok) + + return toks, nil +} + +// Attempts to create an compact map/object, +// returns input as is if length of compacted string exceeds 80. +func maybeCompactMapToken(toks hclwrite.Tokens) hclwrite.Tokens { + s := string(toks.Bytes()) + s = strings.ReplaceAll(s, "\"\n", "\",") + s = strings.ReplaceAll(s, "\n", "") + s = strings.Join(strings.Fields(s), " ") + if len(s) > 80 { + return toks + } + t := simpleTokenFromString(s) + return hclwrite.Tokens{&t} } func simpleTokenFromString(str string) hclwrite.Token { @@ -482,3 +495,11 @@ func orderKeys[T any](settings map[string]T) []string { sort.Strings(keys) return keys } + +func tokensForValue(val cty.Value) hclwrite.Tokens { + toks := hclwrite.TokensForValue(val) + if val.Type().IsMapType() || val.Type().IsObjectType() { + toks = maybeCompactMapToken(toks) + } + return toks +} From 52f70b5d8281ed62bfde7310db9a95f5947c2711 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Thu, 23 Mar 2023 13:18:14 -0500 Subject: [PATCH 051/100] Refactoring to support edge-tracking in graph - Re-implement `modReference` and `varReference` structs to implement a new interface named `reference` - Adopt the `reference` interface in the `ModConnection` struct that tracks edges within the blueprint graph --- pkg/config/config.go | 18 +++--- pkg/config/config_test.go | 34 +++++++--- pkg/config/expand.go | 131 +++++++++++++++++++++++++------------- pkg/config/expand_test.go | 93 +++++++++++++++------------ 4 files changed, 171 insertions(+), 105 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 166c46fd4f..07f6fe1ddb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -271,11 +271,8 @@ const ( // ModConnection defines details about connections between modules. Currently, // only modules connected with "use" are tracked. type ModConnection struct { - toID string - fromID string - // Currently only supports useConnection - kind ConnectionKind - // List of variables shared from module `fromID` to module `toID` + ref reference + kind ConnectionKind sharedVariables []string } @@ -322,7 +319,8 @@ func (dc *DeploymentConfig) listUnusedModules() map[string][]string { unusedModules := make(map[string][]string) for _, conn := range dc.moduleConnections { if conn.isEmpty() { - unusedModules[conn.fromID] = append(unusedModules[conn.fromID], conn.toID) + fromMod := conn.ref.getFromModuleID() + unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.getToModuleID()) } } return unusedModules @@ -508,8 +506,8 @@ func modToGrp(groups []DeploymentGroup, modID string) (int, error) { // checkUsedModuleNames verifies that any used modules have valid names and // are in the correct group -func checkUsedModuleNames(depGroups []DeploymentGroup) error { - for _, grp := range depGroups { +func checkUsedModuleNames(bp Blueprint) error { + for _, grp := range bp.DeploymentGroups { for _, mod := range grp.Modules { for _, usedMod := range mod.Use { ref, err := identifyModuleByReference(usedMod, grp) @@ -517,7 +515,7 @@ func checkUsedModuleNames(depGroups []DeploymentGroup) error { return err } - if err = ref.validate(depGroups); err != nil { + if err = ref.validate(bp); err != nil { return err } @@ -570,7 +568,7 @@ func (dc *DeploymentConfig) validateConfig() { if err = checkModuleAndGroupNames(dc.Config.DeploymentGroups); err != nil { log.Fatal(err) } - if err = checkUsedModuleNames(dc.Config.DeploymentGroups); err != nil { + if err = checkUsedModuleNames(dc.Config); err != nil { log.Fatal(err) } if err = checkBackends(dc.Config); err != nil { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 02747d6c66..c30540cdca 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -400,10 +400,15 @@ func (s *MySuite) TestListUnusedModules(c *C) { got := dc.listUnusedModules() c.Assert(got, HasLen, 0) - // All "use" modules actually used + // test used module with shared variables usedConn := ModConnection{ - toID: "usedModule", - fromID: "usingModule", + ref: modReference{ + ToModuleID: "usedModule", + FromModuleID: "usingModule", + ToGroupID: "group1", + FromGroupID: "group1", + Explicit: true, + }, kind: useConnection, sharedVariables: []string{"var1"}, } @@ -411,10 +416,15 @@ func (s *MySuite) TestListUnusedModules(c *C) { got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 0) - // One fully unused module + // test used module with no shared variables (i.e. "unused") unusedConn := ModConnection{ - toID: "usedModule", - fromID: "usingModule", + ref: modReference{ + ToModuleID: "firstUnusedModule", + FromModuleID: "usingModule", + ToGroupID: "group1", + FromGroupID: "group1", + Explicit: true, + }, kind: useConnection, sharedVariables: []string{}, } @@ -422,17 +432,21 @@ func (s *MySuite) TestListUnusedModules(c *C) { got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 1) - // Two fully unused modules + // test second used module with no shared variables (i.e. "unused") secondUnusedConn := ModConnection{ - toID: "secondUnusedModule", - fromID: "usingModule", + ref: modReference{ + ToModuleID: "secondUnusedModule", + FromModuleID: "usingModule", + ToGroupID: "group1", + FromGroupID: "group1", + Explicit: true, + }, kind: useConnection, sharedVariables: []string{}, } dc.moduleConnections = append(dc.moduleConnections, secondUnusedConn) got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 2) - } func (s *MySuite) TestAddKindToModules(c *C) { diff --git a/pkg/config/expand.go b/pkg/config/expand.go index f338fa01a6..6554906f13 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -275,7 +275,7 @@ func (dc *DeploymentConfig) applyUseModules() error { // this module contains information about the target module that // was specified by the user in the blueprint - toMod, err := toGroup.getModuleByID(modRef.ID) + toMod, err := toGroup.getModuleByID(modRef.ToModuleID) if err != nil { return err } @@ -300,8 +300,7 @@ func (dc *DeploymentConfig) applyUseModules() error { return err } connection := ModConnection{ - toID: toModID, - fromID: fromMod.ID, + ref: modRef, kind: useConnection, sharedVariables: usedVars, } @@ -513,20 +512,46 @@ type varContext struct { blueprint Blueprint } +type reference interface { + validate(Blueprint) error + isIntergroup() bool + String() string + getFromModuleID() string + getToModuleID() string +} + /* A module reference is made by the use keyword and is subject to IGC constraints of references (ordering, explicitness). It has the following fields: - - ID: a module ID + - ToModuleID: the target module ID + - FromModuleID: the source module ID - ToGroupID: the deployment group in which the module is *expected* to be found - FromGroupID: the deployment group from which the reference is made - Explicit: a boolean value indicating whether the user made a reference that explicitly identified ToGroupID rather than inferring it using FromGroupID */ type modReference struct { - ID string - ToGroupID string - FromGroupID string - Explicit bool + ToModuleID string + FromModuleID string + ToGroupID string + FromGroupID string + Explicit bool +} + +func (ref modReference) String() string { + return ref.ToGroupID + "." + ref.ToModuleID +} + +func (ref modReference) isIntergroup() bool { + return ref.ToGroupID == ref.FromGroupID +} + +func (ref modReference) getFromModuleID() string { + return ref.FromModuleID +} + +func (ref modReference) getToModuleID() string { + return ref.FromModuleID } /* @@ -543,12 +568,12 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef modComponents := strings.Split(yamlReference, ".") switch len(modComponents) { case 1: - ref.ID = modComponents[0] + ref.ToModuleID = modComponents[0] ref.ToGroupID = dg.Name ref.FromGroupID = dg.Name case 2: ref.ToGroupID = modComponents[0] - ref.ID = modComponents[1] + ref.ToModuleID = modComponents[1] ref.FromGroupID = dg.Name ref.Explicit = true } @@ -557,7 +582,7 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef // for now check that no fields are the empty string; due to the default // zero values for strings in the "ref" struct, this will also cover the // case that modComponents has wrong # of fields - if ref.ID == "" || ref.ToGroupID == "" || ref.FromGroupID == "" { + if ref.ToModuleID == "" || ref.ToGroupID == "" || ref.FromGroupID == "" { return ref, fmt.Errorf("%s: %s, expected %s", errorMessages["invalidMod"], yamlReference, expectedModFormat) } @@ -567,19 +592,37 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef /* A variable reference has the following fields - - ID: a module ID or "vars" if referring to a deployment variable - Name: the name of the module output or deployment variable + - ToModuleID: the target module ID or "vars" if referring to a deployment variable + - FromModuleID: the source module ID - ToGroupID: the deployment group in which the module is *expected* to be found - FromGroupID: the deployment group from which the reference is made - Explicit: a boolean value indicating whether the user made a reference that explicitly identified ToGroupID rather than inferring it using FromGroupID */ type varReference struct { - ID string - ToGroupID string - FromGroupID string - Name string - Explicit bool + Name string + ToModuleID string + FromModuleID string + ToGroupID string + FromGroupID string + Explicit bool +} + +func (ref varReference) String() string { + return ref.ToGroupID + "." + ref.ToModuleID + "." + ref.Name +} + +func (ref varReference) isIntergroup() bool { + return ref.ToGroupID == ref.FromGroupID +} + +func (ref varReference) getFromModuleID() string { + return ref.FromModuleID +} + +func (ref varReference) getToModuleID() string { + return ref.FromModuleID } /* @@ -599,17 +642,17 @@ func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varRefere // intra-group references length 2 and inter-group references length 3 switch len(varComponents) { case 2: - ref.ID = varComponents[0] + ref.ToModuleID = varComponents[0] ref.Name = varComponents[1] - if ref.ID == "vars" { + if ref.ToModuleID == "vars" { ref.ToGroupID = "deployment" } else { ref.ToGroupID = dg.Name } case 3: ref.ToGroupID = varComponents[0] - ref.ID = varComponents[1] + ref.ToModuleID = varComponents[1] ref.Name = varComponents[2] ref.Explicit = true } @@ -618,24 +661,24 @@ func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varRefere // for now check that source and name are not empty strings; due to the // default zero values for strings in the "ref" struct, this will also // cover the case that varComponents has wrong # of fields - if ref.FromGroupID == "" || ref.ToGroupID == "" || ref.ID == "" || ref.Name == "" { + if ref.FromGroupID == "" || ref.ToGroupID == "" || ref.ToModuleID == "" || ref.Name == "" { return varReference{}, fmt.Errorf("%s %s, expected format: %s", errorMessages["invalidVar"], yamlReference, expectedVarFormat) } return ref, nil } -func (ref *modReference) validate(depGroups []DeploymentGroup) error { - callingModuleGroupIndex := slices.IndexFunc(depGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) +func (ref modReference) validate(bp Blueprint) error { + callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) if callingModuleGroupIndex == -1 { return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.FromGroupID) } - targetModuleGroupIndex, err := modToGrp(depGroups, ref.ID) + targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.ToModuleID) if err != nil { return err } - targetModuleGroupName := depGroups[targetModuleGroupIndex].Name + targetModuleGroupName := bp.DeploymentGroups[targetModuleGroupIndex].Name // Ensure module is from the correct group isInterGroupReference := callingModuleGroupIndex != targetModuleGroupIndex @@ -645,12 +688,12 @@ func (ref *modReference) validate(depGroups []DeploymentGroup) error { if isInterGroupReference { if isRefToLaterGroup { return fmt.Errorf("%s: %s is in a later group", - errorMessages["intergroupOrder"], ref.ID) + errorMessages["intergroupOrder"], ref.ToModuleID) } if !ref.Explicit { return fmt.Errorf("%s: %s must specify a group ID before the module ID", - errorMessages["intergroupImplicit"], ref.ID) + errorMessages["intergroupImplicit"], ref.ToModuleID) } } @@ -659,7 +702,7 @@ func (ref *modReference) validate(depGroups []DeploymentGroup) error { // error after enforcing explicitness of intergroup references if !isCorrectToGroup { return fmt.Errorf("%s: %s.%s", - errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ID) + errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ToModuleID) } return nil @@ -673,26 +716,26 @@ func (ref *modReference) validate(depGroups []DeploymentGroup) error { // ref.ExplicitInterGroup: intergroup references must explicitly identify the // target group ID and intragroup references cannot have an incorrect explicit // group ID -func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]interface{}) error { +func (ref *varReference) validate(bp Blueprint) error { // simplest case to evaluate is a deployment variable's existence if ref.ToGroupID == "deployment" { - if ref.ID == "vars" { - if _, ok := vars[ref.Name]; !ok { + if ref.ToModuleID == "vars" { + if _, ok := bp.Vars[ref.Name]; !ok { return fmt.Errorf("%s: %s is not a deployment variable", errorMessages["varNotFound"], ref.Name) } return nil } - return fmt.Errorf("%s: %s", errorMessages["invalidDeploymentRef"], ref.ID) + return fmt.Errorf("%s: %s", errorMessages["invalidDeploymentRef"], ref) } - targetModuleGroupIndex, err := modToGrp(depGroups, ref.ID) + targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.ToModuleID) if err != nil { return err } - targetModuleGroup := depGroups[targetModuleGroupIndex] + targetModuleGroup := bp.DeploymentGroups[targetModuleGroupIndex] - callingModuleGroupIndex := slices.IndexFunc(depGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) + callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) if callingModuleGroupIndex == -1 { return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.FromGroupID) } @@ -707,12 +750,12 @@ func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]i if isInterGroupReference { if isRefToLaterGroup { return fmt.Errorf("%s: %s is in the later group %s", - errorMessages["intergroupOrder"], ref.ID, ref.ToGroupID) + errorMessages["intergroupOrder"], ref.ToModuleID, ref.ToGroupID) } if !ref.Explicit { return fmt.Errorf("%s: %s must specify the group ID %s before the module ID", - errorMessages["intergroupImplicit"], ref.ID, ref.ToGroupID) + errorMessages["intergroupImplicit"], ref.ToModuleID, ref.ToGroupID) } } @@ -721,15 +764,15 @@ func (ref *varReference) validate(depGroups []DeploymentGroup, vars map[string]i // error after enforcing explicitness of intergroup references if !isCorrectToGroup { return fmt.Errorf("%s: %s.%s should be %s.%s", - errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ID, targetModuleGroup.Name, ref.ID) + errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ToModuleID, targetModuleGroup.Name, ref.ToModuleID) } // at this point, we have a valid intragroup or intergroup references to a // module. must now determine whether the output value actually exists in // the module. - refModIndex := slices.IndexFunc(targetModuleGroup.Modules, func(m Module) bool { return m.ID == ref.ID }) + refModIndex := slices.IndexFunc(targetModuleGroup.Modules, func(m Module) bool { return m.ID == ref.ToModuleID }) if refModIndex == -1 { - log.Fatalf("Could not find module %s", ref.ID) + log.Fatalf("Could not find module %s", ref.ToModuleID) } refMod := targetModuleGroup.Modules[refModIndex] modInfo, err := modulereader.GetModuleInfo(refMod.Source, refMod.Kind) @@ -765,7 +808,7 @@ func expandSimpleVariable(context varContext) (string, error) { return "", err } - if err := varRef.validate(context.blueprint.DeploymentGroups, context.blueprint.Vars); err != nil { + if err := varRef.validate(context.blueprint); err != nil { return "", err } @@ -776,7 +819,7 @@ func expandSimpleVariable(context varContext) (string, error) { expandedVariable = fmt.Sprintf("((var.%s))", varRef.Name) case varRef.FromGroupID: // intragroup reference can make direct reference to module output - expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.ID, varRef.Name) + expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.ToModuleID, varRef.Name) default: // intergroup reference; begin by finding the target module in blueprint @@ -788,9 +831,9 @@ func expandSimpleVariable(context varContext) (string, error) { return "", fmt.Errorf("invalid group reference: %s", varRef.ToGroupID) } toGrp := context.blueprint.DeploymentGroups[toGrpIdx] - toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.ID }) + toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.ToModuleID }) if toModIdx == -1 { - return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.ID) + return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.ToModuleID) } toMod := toGrp.Modules[toModIdx] diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 94f83ab86f..9fdb974567 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -555,21 +555,21 @@ func (s *MySuite) TestIdentifyModuleByReference(c *C) { c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, dg.Name) c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ID, Equals, "module_id") + c.Assert(ref.ToModuleID, Equals, "module_id") c.Assert(ref.Explicit, Equals, false) ref, err = identifyModuleByReference("explicit_group_id.module_id", dg) c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, "explicit_group_id") c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ID, Equals, "module_id") + c.Assert(ref.ToModuleID, Equals, "module_id") c.Assert(ref.Explicit, Equals, true) ref, err = identifyModuleByReference(fmt.Sprintf("%s.module_id", dg.Name), dg) c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, dg.Name) c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ID, Equals, "module_id") + c.Assert(ref.ToModuleID, Equals, "module_id") c.Assert(ref.Explicit, Equals, true) ref, err = identifyModuleByReference("explicit_group_id.module_id.output_name", dg) @@ -605,68 +605,79 @@ func (s *MySuite) TestValidateModuleReference(c *C) { }, } + bp := Blueprint{ + DeploymentGroups: dg, + } + // An intragroup reference from group 0 to module B in 0 (good) ref0ToB0 := modReference{ - ID: dg[0].Modules[1].ID, - ToGroupID: dg[0].Name, - FromGroupID: dg[0].Name, - Explicit: false, + ToModuleID: dg[0].Modules[1].ID, + FromModuleID: "", + ToGroupID: dg[0].Name, + FromGroupID: dg[0].Name, + Explicit: false, } - c.Assert(ref0ToB0.validate(dg), IsNil) + c.Assert(ref0ToB0.validate(bp), IsNil) // An explicit intergroup reference from group 1 to module A in 0 (good) xRef1ToA0 := modReference{ - ID: dg[0].Modules[0].ID, - ToGroupID: dg[0].Name, - FromGroupID: dg[1].Name, - Explicit: true, + ToModuleID: dg[0].Modules[0].ID, + FromModuleID: "", + ToGroupID: dg[0].Name, + FromGroupID: dg[1].Name, + Explicit: true, } - c.Assert(xRef1ToA0.validate(dg), IsNil) + c.Assert(xRef1ToA0.validate(bp), IsNil) // An implicit intergroup reference from group 1 to module A in 0 (bad due to implicit) iRef1ToA0 := modReference{ - ID: dg[0].Modules[0].ID, - ToGroupID: dg[0].Name, - FromGroupID: dg[1].Name, - Explicit: false, + ToModuleID: dg[0].Modules[0].ID, + FromModuleID: "", + ToGroupID: dg[0].Name, + FromGroupID: dg[1].Name, + Explicit: false, } - c.Assert(iRef1ToA0.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupImplicit"])) + c.Assert(iRef1ToA0.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupImplicit"])) // An explicit intergroup reference from group 0 to module 1 in 1 (bad due to group ordering) xRefA0To1 := modReference{ - ID: dg[1].Modules[0].ID, - ToGroupID: dg[1].Name, - FromGroupID: dg[0].Name, - Explicit: true, + ToModuleID: dg[1].Modules[0].ID, + FromModuleID: "", + ToGroupID: dg[1].Name, + FromGroupID: dg[0].Name, + Explicit: true, } - c.Assert(xRefA0To1.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupOrder"])) + c.Assert(xRefA0To1.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupOrder"])) // An explicit intergroup reference from group 0 to B0 with a bad Group ID badRef0ToB0 := modReference{ - ID: dg[0].Modules[1].ID, - ToGroupID: dg[1].Name, - FromGroupID: dg[0].Name, - Explicit: true, + ToModuleID: dg[0].Modules[1].ID, + FromModuleID: "", + ToGroupID: dg[1].Name, + FromGroupID: dg[0].Name, + Explicit: true, } - c.Assert(badRef0ToB0.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["referenceWrongGroup"])) + c.Assert(badRef0ToB0.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["referenceWrongGroup"])) // A target module that doesn't exist (bad) badTargetMod := modReference{ - ID: "bad-module", - ToGroupID: dg[0].Name, - FromGroupID: dg[0].Name, - Explicit: true, + ToModuleID: "bad-module", + FromModuleID: "", + ToGroupID: dg[0].Name, + FromGroupID: dg[0].Name, + Explicit: true, } - c.Assert(badTargetMod.validate(dg), ErrorMatches, fmt.Sprintf("module bad-module was not found")) + c.Assert(badTargetMod.validate(bp), ErrorMatches, "module bad-module was not found") // A source group ID that doesn't exist (bad) badSourceGroup := modReference{ - ID: dg[0].Modules[0].ID, - ToGroupID: dg[0].Name, - FromGroupID: "bad-group", - Explicit: true, + ToModuleID: dg[0].Modules[0].ID, + FromModuleID: "", + ToGroupID: dg[0].Name, + FromGroupID: "bad-group", + Explicit: true, } - c.Assert(badSourceGroup.validate(dg), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["groupNotFound"])) + c.Assert(badSourceGroup.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["groupNotFound"])) } func (s *MySuite) TestIdentifySimpleVariable(c *C) { @@ -681,7 +692,7 @@ func (s *MySuite) TestIdentifySimpleVariable(c *C) { c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, "group_id") c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ID, Equals, "module_id") + c.Assert(ref.ToModuleID, Equals, "module_id") c.Assert(ref.Name, Equals, "output_name") c.Assert(ref.Explicit, Equals, true) @@ -689,7 +700,7 @@ func (s *MySuite) TestIdentifySimpleVariable(c *C) { c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, "calling_group_id") c.Assert(ref.FromGroupID, Equals, "calling_group_id") - c.Assert(ref.ID, Equals, "module_id") + c.Assert(ref.ToModuleID, Equals, "module_id") c.Assert(ref.Name, Equals, "output_name") c.Assert(ref.Explicit, Equals, false) @@ -697,7 +708,7 @@ func (s *MySuite) TestIdentifySimpleVariable(c *C) { c.Assert(err, IsNil) c.Assert(ref.ToGroupID, Equals, "deployment") c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ID, Equals, "vars") + c.Assert(ref.ToModuleID, Equals, "vars") c.Assert(ref.Name, Equals, "variable_name") c.Assert(ref.Explicit, Equals, false) From 701af72a2dcc3bf1de28b711ee35e97ceae60059 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Thu, 23 Mar 2023 15:24:02 -0500 Subject: [PATCH 052/100] Address feedback in #1078 --- pkg/config/config.go | 6 +- pkg/config/config_test.go | 30 +++--- pkg/config/expand.go | 200 +++++++++++++++++++------------------- pkg/config/expand_test.go | 124 +++++++++++------------ 4 files changed, 180 insertions(+), 180 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 07f6fe1ddb..a8f6bf37c0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -319,8 +319,8 @@ func (dc *DeploymentConfig) listUnusedModules() map[string][]string { unusedModules := make(map[string][]string) for _, conn := range dc.moduleConnections { if conn.isEmpty() { - fromMod := conn.ref.getFromModuleID() - unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.getToModuleID()) + fromMod := conn.ref.FromModuleID() + unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.ToModuleID()) } } return unusedModules @@ -520,7 +520,7 @@ func checkUsedModuleNames(bp Blueprint) error { } // TODO: remove this when support is added! - if ref.FromGroupID != ref.ToGroupID { + if ref.fromGroupID != ref.toGroupID { return fmt.Errorf("%s: %s is an intergroup reference", errorMessages["varInAnotherGroup"], usedMod) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c30540cdca..ad4880b5f0 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -403,11 +403,11 @@ func (s *MySuite) TestListUnusedModules(c *C) { // test used module with shared variables usedConn := ModConnection{ ref: modReference{ - ToModuleID: "usedModule", - FromModuleID: "usingModule", - ToGroupID: "group1", - FromGroupID: "group1", - Explicit: true, + toModuleID: "usedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, }, kind: useConnection, sharedVariables: []string{"var1"}, @@ -419,11 +419,11 @@ func (s *MySuite) TestListUnusedModules(c *C) { // test used module with no shared variables (i.e. "unused") unusedConn := ModConnection{ ref: modReference{ - ToModuleID: "firstUnusedModule", - FromModuleID: "usingModule", - ToGroupID: "group1", - FromGroupID: "group1", - Explicit: true, + toModuleID: "firstUnusedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, }, kind: useConnection, sharedVariables: []string{}, @@ -435,11 +435,11 @@ func (s *MySuite) TestListUnusedModules(c *C) { // test second used module with no shared variables (i.e. "unused") secondUnusedConn := ModConnection{ ref: modReference{ - ToModuleID: "secondUnusedModule", - FromModuleID: "usingModule", - ToGroupID: "group1", - FromGroupID: "group1", - Explicit: true, + toModuleID: "secondUnusedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, }, kind: useConnection, sharedVariables: []string{}, diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 6554906f13..8518c70010 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -213,7 +213,7 @@ func useModule( for _, useOutput := range useOutputs { settingName := useOutput.Name - // Explicitly ignore these settings (typically those in blueprint) + // explicitly ignore these settings (typically those in blueprint) if slices.Contains(settingsToIgnore, settingName) { continue } @@ -268,14 +268,14 @@ func (dc *DeploymentConfig) applyUseModules() error { } // to get the module struct, we first needs its group - toGroup, err := dc.getGroupByID(modRef.ToGroupID) + toGroup, err := dc.getGroupByID(modRef.toGroupID) if err != nil { return err } // this module contains information about the target module that // was specified by the user in the blueprint - toMod, err := toGroup.getModuleByID(modRef.ToModuleID) + toMod, err := toGroup.getModuleByID(modRef.toModuleID) if err != nil { return err } @@ -294,7 +294,7 @@ func (dc *DeploymentConfig) applyUseModules() error { // tested but it our unit test infrastructure does not support // running dc.setModulesInfo() on our test configurations toModInfo := dc.ModulesInfo[toGroup.Name][toMod.Source] - usedVars, err := useModule(fromMod, toMod, modRef.ToGroupID, + usedVars, err := useModule(fromMod, toMod, modRef.toGroupID, fromModInfo.Inputs, toModInfo.Outputs, settingsInBlueprint) if err != nil { return err @@ -514,44 +514,44 @@ type varContext struct { type reference interface { validate(Blueprint) error - isIntergroup() bool + IsIntergroup() bool String() string - getFromModuleID() string - getToModuleID() string + FromModuleID() string + ToModuleID() string } /* A module reference is made by the use keyword and is subject to IGC constraints of references (ordering, explicitness). It has the following fields: - - ToModuleID: the target module ID - - FromModuleID: the source module ID - - ToGroupID: the deployment group in which the module is *expected* to be found - - FromGroupID: the deployment group from which the reference is made - - Explicit: a boolean value indicating whether the user made a reference that - explicitly identified ToGroupID rather than inferring it using FromGroupID + - toModuleID: the target module ID + - fromModuleID: the source module ID + - toGroupID: the deployment group in which the module is *expected* to be found + - fromGroupID: the deployment group from which the reference is made + - explicit: a boolean value indicating whether the user made a reference that + explicitly identified toGroupID rather than inferring it using fromGroupID */ type modReference struct { - ToModuleID string - FromModuleID string - ToGroupID string - FromGroupID string - Explicit bool + toModuleID string + fromModuleID string + toGroupID string + fromGroupID string + explicit bool } func (ref modReference) String() string { - return ref.ToGroupID + "." + ref.ToModuleID + return ref.toGroupID + "." + ref.toModuleID } -func (ref modReference) isIntergroup() bool { - return ref.ToGroupID == ref.FromGroupID +func (ref modReference) IsIntergroup() bool { + return ref.toGroupID == ref.fromGroupID } -func (ref modReference) getFromModuleID() string { - return ref.FromModuleID +func (ref modReference) FromModuleID() string { + return ref.fromModuleID } -func (ref modReference) getToModuleID() string { - return ref.FromModuleID +func (ref modReference) ToModuleID() string { + return ref.toModuleID } /* @@ -568,21 +568,21 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef modComponents := strings.Split(yamlReference, ".") switch len(modComponents) { case 1: - ref.ToModuleID = modComponents[0] - ref.ToGroupID = dg.Name - ref.FromGroupID = dg.Name + ref.toModuleID = modComponents[0] + ref.toGroupID = dg.Name + ref.fromGroupID = dg.Name case 2: - ref.ToGroupID = modComponents[0] - ref.ToModuleID = modComponents[1] - ref.FromGroupID = dg.Name - ref.Explicit = true + ref.toGroupID = modComponents[0] + ref.toModuleID = modComponents[1] + ref.fromGroupID = dg.Name + ref.explicit = true } // should consider more sophisticated definition of valid values here. // for now check that no fields are the empty string; due to the default // zero values for strings in the "ref" struct, this will also cover the // case that modComponents has wrong # of fields - if ref.ToModuleID == "" || ref.ToGroupID == "" || ref.FromGroupID == "" { + if ref.toModuleID == "" || ref.toGroupID == "" || ref.fromGroupID == "" { return ref, fmt.Errorf("%s: %s, expected %s", errorMessages["invalidMod"], yamlReference, expectedModFormat) } @@ -593,36 +593,36 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef /* A variable reference has the following fields - Name: the name of the module output or deployment variable - - ToModuleID: the target module ID or "vars" if referring to a deployment variable - - FromModuleID: the source module ID - - ToGroupID: the deployment group in which the module is *expected* to be found - - FromGroupID: the deployment group from which the reference is made - - Explicit: a boolean value indicating whether the user made a reference that - explicitly identified ToGroupID rather than inferring it using FromGroupID + - toModuleID: the target module ID or "vars" if referring to a deployment variable + - fromModuleID: the source module ID + - toGroupID: the deployment group in which the module is *expected* to be found + - fromGroupID: the deployment group from which the reference is made + - explicit: a boolean value indicating whether the user made a reference that + explicitly identified toGroupID rather than inferring it using fromGroupID */ type varReference struct { - Name string - ToModuleID string - FromModuleID string - ToGroupID string - FromGroupID string - Explicit bool + name string + toModuleID string + fromModuleID string + toGroupID string + fromGroupID string + explicit bool } func (ref varReference) String() string { - return ref.ToGroupID + "." + ref.ToModuleID + "." + ref.Name + return ref.toGroupID + "." + ref.toModuleID + "." + ref.name } -func (ref varReference) isIntergroup() bool { - return ref.ToGroupID == ref.FromGroupID +func (ref varReference) IsIntergroup() bool { + return ref.toGroupID == ref.fromGroupID } -func (ref varReference) getFromModuleID() string { - return ref.FromModuleID +func (ref varReference) FromModuleID() string { + return ref.fromModuleID } -func (ref varReference) getToModuleID() string { - return ref.FromModuleID +func (ref varReference) ToModuleID() string { + return ref.toModuleID } /* @@ -637,31 +637,31 @@ func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varRefere // struct defaults: empty strings and false booleans var ref varReference - ref.FromGroupID = dg.Name + ref.fromGroupID = dg.Name // intra-group references length 2 and inter-group references length 3 switch len(varComponents) { case 2: - ref.ToModuleID = varComponents[0] - ref.Name = varComponents[1] + ref.toModuleID = varComponents[0] + ref.name = varComponents[1] - if ref.ToModuleID == "vars" { - ref.ToGroupID = "deployment" + if ref.toModuleID == "vars" { + ref.toGroupID = "deployment" } else { - ref.ToGroupID = dg.Name + ref.toGroupID = dg.Name } case 3: - ref.ToGroupID = varComponents[0] - ref.ToModuleID = varComponents[1] - ref.Name = varComponents[2] - ref.Explicit = true + ref.toGroupID = varComponents[0] + ref.toModuleID = varComponents[1] + ref.name = varComponents[2] + ref.explicit = true } // should consider more sophisticated definition of valid values here. // for now check that source and name are not empty strings; due to the // default zero values for strings in the "ref" struct, this will also // cover the case that varComponents has wrong # of fields - if ref.FromGroupID == "" || ref.ToGroupID == "" || ref.ToModuleID == "" || ref.Name == "" { + if ref.fromGroupID == "" || ref.toGroupID == "" || ref.toModuleID == "" || ref.name == "" { return varReference{}, fmt.Errorf("%s %s, expected format: %s", errorMessages["invalidVar"], yamlReference, expectedVarFormat) } @@ -669,12 +669,12 @@ func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varRefere } func (ref modReference) validate(bp Blueprint) error { - callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) + callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.fromGroupID }) if callingModuleGroupIndex == -1 { - return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.FromGroupID) + return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.fromGroupID) } - targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.ToModuleID) + targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.toModuleID) if err != nil { return err } @@ -683,17 +683,17 @@ func (ref modReference) validate(bp Blueprint) error { // Ensure module is from the correct group isInterGroupReference := callingModuleGroupIndex != targetModuleGroupIndex isRefToLaterGroup := targetModuleGroupIndex > callingModuleGroupIndex - isCorrectToGroup := ref.ToGroupID == targetModuleGroupName + isCorrectToGroup := ref.toGroupID == targetModuleGroupName if isInterGroupReference { if isRefToLaterGroup { return fmt.Errorf("%s: %s is in a later group", - errorMessages["intergroupOrder"], ref.ToModuleID) + errorMessages["intergroupOrder"], ref.toModuleID) } - if !ref.Explicit { + if !ref.explicit { return fmt.Errorf("%s: %s must specify a group ID before the module ID", - errorMessages["intergroupImplicit"], ref.ToModuleID) + errorMessages["intergroupImplicit"], ref.toModuleID) } } @@ -702,7 +702,7 @@ func (ref modReference) validate(bp Blueprint) error { // error after enforcing explicitness of intergroup references if !isCorrectToGroup { return fmt.Errorf("%s: %s.%s", - errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ToModuleID) + errorMessages["referenceWrongGroup"], ref.toGroupID, ref.toModuleID) } return nil @@ -712,50 +712,50 @@ func (ref modReference) validate(bp Blueprint) error { // the reference must be to the same or earlier group. // ref.GroupID: this group must exist or be the value "deployment" // ref.ID: must be an existing module ID or "vars" (if groupID is "deployment") -// ref.Name: must match a module output name or deployment variable name -// ref.ExplicitInterGroup: intergroup references must explicitly identify the +// ref.name: must match a module output name or deployment variable name +// ref.explicitInterGroup: intergroup references must explicitly identify the // target group ID and intragroup references cannot have an incorrect explicit // group ID func (ref *varReference) validate(bp Blueprint) error { // simplest case to evaluate is a deployment variable's existence - if ref.ToGroupID == "deployment" { - if ref.ToModuleID == "vars" { - if _, ok := bp.Vars[ref.Name]; !ok { + if ref.toGroupID == "deployment" { + if ref.toModuleID == "vars" { + if _, ok := bp.Vars[ref.name]; !ok { return fmt.Errorf("%s: %s is not a deployment variable", - errorMessages["varNotFound"], ref.Name) + errorMessages["varNotFound"], ref.name) } return nil } return fmt.Errorf("%s: %s", errorMessages["invalidDeploymentRef"], ref) } - targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.ToModuleID) + targetModuleGroupIndex, err := modToGrp(bp.DeploymentGroups, ref.toModuleID) if err != nil { return err } targetModuleGroup := bp.DeploymentGroups[targetModuleGroupIndex] - callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.FromGroupID }) + callingModuleGroupIndex := slices.IndexFunc(bp.DeploymentGroups, func(d DeploymentGroup) bool { return d.Name == ref.fromGroupID }) if callingModuleGroupIndex == -1 { - return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.FromGroupID) + return fmt.Errorf("%s: %s", errorMessages["groupNotFound"], ref.fromGroupID) } // at this point, we know the target module exists. now record whether it // is intergroup and whether it comes in a (disallowed) later group isInterGroupReference := targetModuleGroupIndex != callingModuleGroupIndex isRefToLaterGroup := targetModuleGroupIndex > callingModuleGroupIndex - isCorrectToGroup := ref.ToGroupID == targetModuleGroup.Name + isCorrectToGroup := ref.toGroupID == targetModuleGroup.Name // intergroup references must be explicit about group and refer to an earlier group; if isInterGroupReference { if isRefToLaterGroup { return fmt.Errorf("%s: %s is in the later group %s", - errorMessages["intergroupOrder"], ref.ToModuleID, ref.ToGroupID) + errorMessages["intergroupOrder"], ref.toModuleID, ref.toGroupID) } - if !ref.Explicit { + if !ref.explicit { return fmt.Errorf("%s: %s must specify the group ID %s before the module ID", - errorMessages["intergroupImplicit"], ref.ToModuleID, ref.ToGroupID) + errorMessages["intergroupImplicit"], ref.toModuleID, ref.toGroupID) } } @@ -764,15 +764,15 @@ func (ref *varReference) validate(bp Blueprint) error { // error after enforcing explicitness of intergroup references if !isCorrectToGroup { return fmt.Errorf("%s: %s.%s should be %s.%s", - errorMessages["referenceWrongGroup"], ref.ToGroupID, ref.ToModuleID, targetModuleGroup.Name, ref.ToModuleID) + errorMessages["referenceWrongGroup"], ref.toGroupID, ref.toModuleID, targetModuleGroup.Name, ref.toModuleID) } // at this point, we have a valid intragroup or intergroup references to a // module. must now determine whether the output value actually exists in // the module. - refModIndex := slices.IndexFunc(targetModuleGroup.Modules, func(m Module) bool { return m.ID == ref.ToModuleID }) + refModIndex := slices.IndexFunc(targetModuleGroup.Modules, func(m Module) bool { return m.ID == ref.toModuleID }) if refModIndex == -1 { - log.Fatalf("Could not find module %s", ref.ToModuleID) + log.Fatalf("Could not find module %s", ref.toModuleID) } refMod := targetModuleGroup.Modules[refModIndex] modInfo, err := modulereader.GetModuleInfo(refMod.Source, refMod.Kind) @@ -781,10 +781,10 @@ func (ref *varReference) validate(bp Blueprint) error { "failed to get info for module at %s while expanding variables: %e", refMod.Source, err) } - found := slices.ContainsFunc(modInfo.Outputs, func(o modulereader.VarInfo) bool { return o.Name == ref.Name }) + found := slices.ContainsFunc(modInfo.Outputs, func(o modulereader.VarInfo) bool { return o.Name == ref.name }) if !found { return fmt.Errorf("%s: module %s did not have output %s", - errorMessages["noOutput"], refMod.ID, ref.Name) + errorMessages["noOutput"], refMod.ID, ref.name) } return nil @@ -813,37 +813,37 @@ func expandSimpleVariable(context varContext) (string, error) { } var expandedVariable string - switch varRef.ToGroupID { + switch varRef.toGroupID { case "deployment": // deployment variables - expandedVariable = fmt.Sprintf("((var.%s))", varRef.Name) - case varRef.FromGroupID: + expandedVariable = fmt.Sprintf("((var.%s))", varRef.name) + case varRef.fromGroupID: // intragroup reference can make direct reference to module output - expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.ToModuleID, varRef.Name) + expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.toModuleID, varRef.name) default: // intergroup reference; begin by finding the target module in blueprint toGrpIdx := slices.IndexFunc( context.blueprint.DeploymentGroups, - func(g DeploymentGroup) bool { return g.Name == varRef.ToGroupID }) + func(g DeploymentGroup) bool { return g.Name == varRef.toGroupID }) if toGrpIdx == -1 { - return "", fmt.Errorf("invalid group reference: %s", varRef.ToGroupID) + return "", fmt.Errorf("invalid group reference: %s", varRef.toGroupID) } toGrp := context.blueprint.DeploymentGroups[toGrpIdx] - toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.ToModuleID }) + toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.toModuleID }) if toModIdx == -1 { - return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.ToModuleID) + return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.toModuleID) } toMod := toGrp.Modules[toModIdx] // ensure that the target module outputs the value in the root module // state and not just internally within its deployment group - if !slices.Contains(toMod.Outputs, varRef.Name) { - toMod.Outputs = append(toMod.Outputs, varRef.Name) + if !slices.Contains(toMod.Outputs, varRef.name) { + toMod.Outputs = append(toMod.Outputs, varRef.name) } - // TODO: expandedVariable = fmt.Sprintf("((var.%s_%s))", ref.Name, ref.ID) + // TODO: expandedVariable = fmt.Sprintf("((var.%s_%s))", ref.name, ref.ID) return "", fmt.Errorf("%s: %s is an intergroup reference", errorMessages["varInAnotherGroup"], context.varString) } diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 9fdb974567..b7f4f7be1e 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -553,24 +553,24 @@ func (s *MySuite) TestIdentifyModuleByReference(c *C) { ref, err = identifyModuleByReference("module_id", dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, dg.Name) - c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ToModuleID, Equals, "module_id") - c.Assert(ref.Explicit, Equals, false) + c.Assert(ref.toGroupID, Equals, dg.Name) + c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.explicit, Equals, false) ref, err = identifyModuleByReference("explicit_group_id.module_id", dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, "explicit_group_id") - c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ToModuleID, Equals, "module_id") - c.Assert(ref.Explicit, Equals, true) + c.Assert(ref.toGroupID, Equals, "explicit_group_id") + c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.explicit, Equals, true) ref, err = identifyModuleByReference(fmt.Sprintf("%s.module_id", dg.Name), dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, dg.Name) - c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ToModuleID, Equals, "module_id") - c.Assert(ref.Explicit, Equals, true) + c.Assert(ref.toGroupID, Equals, dg.Name) + c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.explicit, Equals, true) ref, err = identifyModuleByReference("explicit_group_id.module_id.output_name", dg) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["invalidMod"])) @@ -611,71 +611,71 @@ func (s *MySuite) TestValidateModuleReference(c *C) { // An intragroup reference from group 0 to module B in 0 (good) ref0ToB0 := modReference{ - ToModuleID: dg[0].Modules[1].ID, - FromModuleID: "", - ToGroupID: dg[0].Name, - FromGroupID: dg[0].Name, - Explicit: false, + toModuleID: dg[0].Modules[1].ID, + fromModuleID: "", + toGroupID: dg[0].Name, + fromGroupID: dg[0].Name, + explicit: false, } c.Assert(ref0ToB0.validate(bp), IsNil) // An explicit intergroup reference from group 1 to module A in 0 (good) xRef1ToA0 := modReference{ - ToModuleID: dg[0].Modules[0].ID, - FromModuleID: "", - ToGroupID: dg[0].Name, - FromGroupID: dg[1].Name, - Explicit: true, + toModuleID: dg[0].Modules[0].ID, + fromModuleID: "", + toGroupID: dg[0].Name, + fromGroupID: dg[1].Name, + explicit: true, } c.Assert(xRef1ToA0.validate(bp), IsNil) // An implicit intergroup reference from group 1 to module A in 0 (bad due to implicit) iRef1ToA0 := modReference{ - ToModuleID: dg[0].Modules[0].ID, - FromModuleID: "", - ToGroupID: dg[0].Name, - FromGroupID: dg[1].Name, - Explicit: false, + toModuleID: dg[0].Modules[0].ID, + fromModuleID: "", + toGroupID: dg[0].Name, + fromGroupID: dg[1].Name, + explicit: false, } c.Assert(iRef1ToA0.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupImplicit"])) // An explicit intergroup reference from group 0 to module 1 in 1 (bad due to group ordering) xRefA0To1 := modReference{ - ToModuleID: dg[1].Modules[0].ID, - FromModuleID: "", - ToGroupID: dg[1].Name, - FromGroupID: dg[0].Name, - Explicit: true, + toModuleID: dg[1].Modules[0].ID, + fromModuleID: "", + toGroupID: dg[1].Name, + fromGroupID: dg[0].Name, + explicit: true, } c.Assert(xRefA0To1.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["intergroupOrder"])) // An explicit intergroup reference from group 0 to B0 with a bad Group ID badRef0ToB0 := modReference{ - ToModuleID: dg[0].Modules[1].ID, - FromModuleID: "", - ToGroupID: dg[1].Name, - FromGroupID: dg[0].Name, - Explicit: true, + toModuleID: dg[0].Modules[1].ID, + fromModuleID: "", + toGroupID: dg[1].Name, + fromGroupID: dg[0].Name, + explicit: true, } c.Assert(badRef0ToB0.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["referenceWrongGroup"])) // A target module that doesn't exist (bad) badTargetMod := modReference{ - ToModuleID: "bad-module", - FromModuleID: "", - ToGroupID: dg[0].Name, - FromGroupID: dg[0].Name, - Explicit: true, + toModuleID: "bad-module", + fromModuleID: "", + toGroupID: dg[0].Name, + fromGroupID: dg[0].Name, + explicit: true, } c.Assert(badTargetMod.validate(bp), ErrorMatches, "module bad-module was not found") // A source group ID that doesn't exist (bad) badSourceGroup := modReference{ - ToModuleID: dg[0].Modules[0].ID, - FromModuleID: "", - ToGroupID: dg[0].Name, - FromGroupID: "bad-group", - Explicit: true, + toModuleID: dg[0].Modules[0].ID, + fromModuleID: "", + toGroupID: dg[0].Name, + fromGroupID: "bad-group", + explicit: true, } c.Assert(badSourceGroup.validate(bp), ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["groupNotFound"])) } @@ -690,27 +690,27 @@ func (s *MySuite) TestIdentifySimpleVariable(c *C) { ref, err = identifySimpleVariable("group_id.module_id.output_name", dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, "group_id") - c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ToModuleID, Equals, "module_id") - c.Assert(ref.Name, Equals, "output_name") - c.Assert(ref.Explicit, Equals, true) + c.Assert(ref.toGroupID, Equals, "group_id") + c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.name, Equals, "output_name") + c.Assert(ref.explicit, Equals, true) ref, err = identifySimpleVariable("module_id.output_name", dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, "calling_group_id") - c.Assert(ref.FromGroupID, Equals, "calling_group_id") - c.Assert(ref.ToModuleID, Equals, "module_id") - c.Assert(ref.Name, Equals, "output_name") - c.Assert(ref.Explicit, Equals, false) + c.Assert(ref.toGroupID, Equals, "calling_group_id") + c.Assert(ref.fromGroupID, Equals, "calling_group_id") + c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.name, Equals, "output_name") + c.Assert(ref.explicit, Equals, false) ref, err = identifySimpleVariable("vars.variable_name", dg) c.Assert(err, IsNil) - c.Assert(ref.ToGroupID, Equals, "deployment") - c.Assert(ref.FromGroupID, Equals, dg.Name) - c.Assert(ref.ToModuleID, Equals, "vars") - c.Assert(ref.Name, Equals, "variable_name") - c.Assert(ref.Explicit, Equals, false) + c.Assert(ref.toGroupID, Equals, "deployment") + c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.toModuleID, Equals, "vars") + c.Assert(ref.name, Equals, "variable_name") + c.Assert(ref.explicit, Equals, false) ref, err = identifySimpleVariable("foo", dg) c.Assert(err, NotNil) From fa3db068c02ff1da61a7b82284ca6e3671d564ab Mon Sep 17 00:00:00 2001 From: Nick Stroud Date: Thu, 23 Mar 2023 09:20:53 -0700 Subject: [PATCH 053/100] Add additional documentation on enable_reconfigure troubleshooting --- .../schedmd-slurm-gcp-v5-controller/README.md | 27 ++++++++++++++----- docs/slurm-troubleshooting.md | 9 +++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md index 5426d24650..a0970c9866 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md @@ -11,11 +11,13 @@ The [user guide][slurm-ug] provides detailed instructions on customizing and enhancing the Slurm on GCP cluster as well as recommendations on configuring the controller for optimal performance at different scales. -> **_WARNING:_** The variables [enable\_reconfigure], [enable\_cleanup\_compute] and -> [enable\_cleanup\_subscriptions], if set to true, require additional -> dependencies **to be installed on the system running `terraform apply`**. -> Python3 (>=3.6.0, <4.0.0) must be installed along with the pip packages listed in the -> [requirements.txt] file of [SchedMD/slurm-gcp]. +> **_WARNING:_** The variables [enable\_reconfigure], +> [enable\_cleanup\_compute] and [enable\_cleanup\_subscriptions], if set to +> true, require additional dependencies **to be installed on the system running +> `terraform apply`**. Python3 (>=3.6.0, <4.0.0) must be installed along with +> the pip packages listed in the [requirements.txt] file of +> [SchedMD/slurm-gcp]. See the +> [documentation below](#live-cluster-reconfiguration-enable_reconfigure). [SchedMD/slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2 [slurm\_controller\_instance]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster/modules/slurm_controller_instance @@ -52,6 +54,7 @@ For a complete example using this module, see [slurm-gcp-v5-cluster.yaml](../../../examples/slurm-gcp-v5-cluster.yaml). ### Live Cluster Reconfiguration (`enable_reconfigure`) + The schedmd-slurm-gcp-v5-controller module supports the reconfiguration of partitions and slurm configuration in a running, active cluster. This option is activated through the `enable_reconfigure` setting: @@ -82,7 +85,19 @@ This option has some additional requirements: * The project in your gcloud config must match the project the cluster is being deployed onto due to a known issue with the reconfigure scripts. To set your default config project, run the following command: - `gcloud config set core/<>` + + ```bash + gcloud config set core/project <> + ``` + + If the gcloud project ID is not properly set you may see an error during + terraform deployment similar to the following: + + ```text + google.api_core.exceptions.NotFound: 404 Resource not found + Could not find in SpannerConfigStore: + TopicByProjectIdAndName(project_id=, topic_name=) + ``` [optdeps]: https://github.com/SchedMD/slurm-gcp/tree/5.6.2/terraform/slurm_cluster#optional diff --git a/docs/slurm-troubleshooting.md b/docs/slurm-troubleshooting.md index dd90aa0d77..3200febd3e 100644 --- a/docs/slurm-troubleshooting.md +++ b/docs/slurm-troubleshooting.md @@ -237,3 +237,12 @@ For example: This is a known issue, the recommended workaround is to use different naming for the `local_mount` and `filestore_share_name`. + +### `local-exec provisioner error` During Terraform Apply + +Using the `enable_reconfigure` setting with Slurm v5 modules uses `local-exec` +provisioners to perform additional cluster configuration. Some common issues +experienced when using this feature are missing local python requirements and +incorrectly configured gcloud cli. There is more information about these issues +and fixes on the +[`schedmd-slurm-gcp-v5-controller` documentation](../community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md#live-cluster-reconfiguration-enable_reconfigure). From 3d29341402aff79c04a06461298ee8d7b219e105 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Thu, 23 Mar 2023 21:14:15 +0000 Subject: [PATCH 054/100] Use stable order while writing terraform.tfvars --- pkg/modulewriter/hcl_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/modulewriter/hcl_utils.go b/pkg/modulewriter/hcl_utils.go index aa84501dd4..90eb7bb68b 100644 --- a/pkg/modulewriter/hcl_utils.go +++ b/pkg/modulewriter/hcl_utils.go @@ -45,9 +45,9 @@ func writeHclAttributes(vars map[string]cty.Value, dst string) error { hclBody := hclFile.Body() // for each variable - for k, v := range vars { + for _, k := range orderKeys(vars) { // Write attribute - hclBody.SetAttributeValue(k, v) + hclBody.SetAttributeValue(k, vars[k]) } // Write file From e4504a5ae3f3383c3bce45a2755bdf3c69cf71cc Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 24 Mar 2023 11:42:34 -0500 Subject: [PATCH 055/100] Refactor moduleConnections as a map Will facilitate searching for connections previously made within a given module. --- pkg/config/config.go | 51 ++++++++++++++++++++++++++-------- pkg/config/config_test.go | 58 ++++++++++++++++----------------------- pkg/config/expand.go | 7 +---- 3 files changed, 63 insertions(+), 53 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index a8f6bf37c0..46f46a2884 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -36,8 +36,9 @@ import ( ) const ( - expectedVarFormat string = "$(vars.var_name) or $(module_id.output_name)" - expectedModFormat string = "$(module_id) or $(group_id.module_id)" + expectedVarFormat string = "$(vars.var_name) or $(module_id.output_name)" + expectedModFormat string = "$(module_id) or $(group_id.module_id)" + unexpectedConnectionKind string = "connectionKind must be useConnection" ) var errorMessages = map[string]string{ @@ -257,22 +258,26 @@ type Blueprint struct { TerraformBackendDefaults TerraformBackend `yaml:"terraform_backend_defaults"` } -// ConnectionKind defines the kind of module connection, defined by the source +// connectionKind defines the kind of module connection, defined by the source // of the connection. Currently, only Use is supported. -type ConnectionKind int +type connectionKind int const ( - undefinedConnection ConnectionKind = iota + undefinedConnection connectionKind = iota useConnection // explicitConnection // globalConnection ) +func (c connectionKind) IsValid() bool { + return c == useConnection +} + // ModConnection defines details about connections between modules. Currently, // only modules connected with "use" are tracked. type ModConnection struct { ref reference - kind ConnectionKind + kind connectionKind sharedVariables []string } @@ -296,7 +301,7 @@ type DeploymentConfig struct { // Indexed by Resource Group name and Module Source ModulesInfo map[string]map[string]modulereader.ModuleInfo expanded bool - moduleConnections []ModConnection + moduleConnections map[string][]ModConnection } // ExpandConfig expands the yaml config in place @@ -313,14 +318,36 @@ func (dc *DeploymentConfig) ExpandConfig() error { return nil } +func (dc *DeploymentConfig) addModuleConnection(ref reference, kind connectionKind, sharedVariables []string) error { + if dc.moduleConnections == nil { + dc.moduleConnections = make(map[string][]ModConnection) + } + + if !kind.IsValid() { + log.Fatal(unexpectedConnectionKind) + } + + conn := ModConnection{ + ref: ref, + kind: kind, + sharedVariables: sharedVariables, + } + + fromModID := ref.FromModuleID() + dc.moduleConnections[fromModID] = append(dc.moduleConnections[fromModID], conn) + return nil +} + // listUnusedModules provides a mapping of modules to modules that are in the // "use" field, but not actually used. func (dc *DeploymentConfig) listUnusedModules() map[string][]string { unusedModules := make(map[string][]string) - for _, conn := range dc.moduleConnections { - if conn.isEmpty() { - fromMod := conn.ref.FromModuleID() - unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.ToModuleID()) + for _, connections := range dc.moduleConnections { + for _, conn := range connections { + if conn.isEmpty() { + fromMod := conn.ref.FromModuleID() + unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.ToModuleID()) + } } } return unusedModules @@ -351,7 +378,7 @@ func NewDeploymentConfig(configFilename string) (DeploymentConfig, error) { newDeploymentConfig = DeploymentConfig{ Config: blueprint, - moduleConnections: []ModConnection{}, + moduleConnections: make(map[string][]ModConnection), } return newDeploymentConfig, nil } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index ad4880b5f0..095d94a60b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -220,7 +220,7 @@ func getDeploymentConfigForTest() DeploymentConfig { testModuleSourceWithLabels: testModuleInfo, }, }, - moduleConnections: []ModConnection{}, + moduleConnections: make(map[string][]ModConnection), } // the next two steps simulate relevant steps in ghpc expand dc.addMetadataToModules() @@ -312,6 +312,7 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { testModuleSource1: testModuleInfo1, }, }, + moduleConnections: make(map[string][]ModConnection), } dc.addMetadataToModules() @@ -400,51 +401,38 @@ func (s *MySuite) TestListUnusedModules(c *C) { got := dc.listUnusedModules() c.Assert(got, HasLen, 0) - // test used module with shared variables - usedConn := ModConnection{ - ref: modReference{ - toModuleID: "usedModule", - fromModuleID: "usingModule", - toGroupID: "group1", - fromGroupID: "group1", - explicit: true, - }, - kind: useConnection, - sharedVariables: []string{"var1"}, + modRef0 := modReference{ + toModuleID: "usedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, } - dc.moduleConnections = []ModConnection{usedConn} + dc.addModuleConnection(modRef0, useConnection, []string{"var1"}) got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 0) // test used module with no shared variables (i.e. "unused") - unusedConn := ModConnection{ - ref: modReference{ - toModuleID: "firstUnusedModule", - fromModuleID: "usingModule", - toGroupID: "group1", - fromGroupID: "group1", - explicit: true, - }, - kind: useConnection, - sharedVariables: []string{}, + modRef1 := modReference{ + toModuleID: "firstUnusedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, } - dc.moduleConnections = append(dc.moduleConnections, unusedConn) + dc.addModuleConnection(modRef1, useConnection, []string{}) got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 1) // test second used module with no shared variables (i.e. "unused") - secondUnusedConn := ModConnection{ - ref: modReference{ - toModuleID: "secondUnusedModule", - fromModuleID: "usingModule", - toGroupID: "group1", - fromGroupID: "group1", - explicit: true, - }, - kind: useConnection, - sharedVariables: []string{}, + modRef2 := modReference{ + toModuleID: "secondUnusedModule", + fromModuleID: "usingModule", + toGroupID: "group1", + fromGroupID: "group1", + explicit: true, } - dc.moduleConnections = append(dc.moduleConnections, secondUnusedConn) + dc.addModuleConnection(modRef2, useConnection, []string{}) got = dc.listUnusedModules() c.Assert(got["usingModule"], HasLen, 2) } diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 8518c70010..2e64858329 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -299,12 +299,7 @@ func (dc *DeploymentConfig) applyUseModules() error { if err != nil { return err } - connection := ModConnection{ - ref: modRef, - kind: useConnection, - sharedVariables: usedVars, - } - dc.moduleConnections = append(dc.moduleConnections, connection) + dc.addModuleConnection(modRef, useConnection, usedVars) } } } From e0a89775ee005a5303f666c04d16d2a206ab5a5d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 24 Mar 2023 11:53:44 -0500 Subject: [PATCH 056/100] Improve version constraint on batch-job-template module --- modules/scheduler/batch-job-template/README.md | 2 +- modules/scheduler/batch-job-template/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/scheduler/batch-job-template/README.md b/modules/scheduler/batch-job-template/README.md index 26864b2a85..a6496cb7a8 100644 --- a/modules/scheduler/batch-job-template/README.md +++ b/modules/scheduler/batch-job-template/README.md @@ -134,7 +134,7 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | > 7.6.0 | +| [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 8.0 | | [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | ## Resources diff --git a/modules/scheduler/batch-job-template/main.tf b/modules/scheduler/batch-job-template/main.tf index 900f4d2f03..bb5876136f 100644 --- a/modules/scheduler/batch-job-template/main.tf +++ b/modules/scheduler/batch-job-template/main.tf @@ -66,7 +66,7 @@ locals { module "instance_template" { source = "terraform-google-modules/vm/google//modules/instance_template" - version = "> 7.6.0" + version = "~> 8.0" name_prefix = var.instance_template == null ? "${local.job_id}-instance-template" : "unused-template" project_id = var.project_id From 886b5e3c878bae25d1914fedb94c80edeb05539f Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Fri, 24 Mar 2023 11:30:25 -0700 Subject: [PATCH 057/100] Remove field Module.ModuleName add DeploymentSource instead (#1081) Remove any source-related logic from tfwritter and packerwriter, simply use DeploymentSource. --- pkg/config/config.go | 5 ++-- pkg/modulewriter/modulewriter.go | 33 +++++++++++++++------------ pkg/modulewriter/modulewriter_test.go | 5 ++-- pkg/modulewriter/packerwriter.go | 2 +- pkg/modulewriter/tfwriter.go | 10 +------- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 46f46a2884..338a286ffe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -226,10 +226,11 @@ func (g DeploymentGroup) HasKind(kind string) bool { // Module stores YAML definition of an HPC cluster component defined in a blueprint type Module struct { - Source string + Source string + // DeploymentSource - is source to be used for this module in written deployment. + DeploymentSource string `yaml:"-"` // "-" prevents user from specifying it Kind string ID string - ModuleName string Use []string WrapSettingsWith map[string][]string Outputs []string `yaml:"outputs,omitempty"` diff --git a/pkg/modulewriter/modulewriter.go b/pkg/modulewriter/modulewriter.go index 61f22c64e8..3a1c2f26c9 100644 --- a/pkg/modulewriter/modulewriter.go +++ b/pkg/modulewriter/modulewriter.go @@ -135,35 +135,38 @@ func createGroupDirs(deploymentPath string, deploymentGroups *[]config.Deploymen func copySource(deploymentPath string, deploymentGroups *[]config.DeploymentGroup) error { - for iGrp, grp := range *deploymentGroups { + for iGrp := range *deploymentGroups { + grp := &(*deploymentGroups)[iGrp] basePath := filepath.Join(deploymentPath, grp.Name) - for iMod, module := range grp.Modules { - if sourcereader.IsGitPath(module.Source) { + for iMod := range grp.Modules { + mod := &grp.Modules[iMod] + if sourcereader.IsGitPath(mod.Source) { + mod.DeploymentSource = mod.Source continue } /* Copy source files */ - moduleName := filepath.Base(module.Source) - (*deploymentGroups)[iGrp].Modules[iMod].ModuleName = moduleName - var destPath string - switch module.Kind { + moduleName := filepath.Base(mod.Source) + var deplSource string + switch mod.Kind { case "terraform": - destPath = filepath.Join(basePath, "modules", moduleName) + deplSource = "./" + filepath.Join("modules", moduleName) case "packer": - destPath = filepath.Join(basePath, module.ID) + deplSource = mod.ID } - _, err := os.Stat(destPath) - if err == nil { + mod.DeploymentSource = deplSource + fullPath := filepath.Join(basePath, deplSource) + if _, err := os.Stat(fullPath); err == nil { continue } - reader := sourcereader.Factory(module.Source) - if err := reader.GetModule(module.Source, destPath); err != nil { - return fmt.Errorf("failed to get module from %s to %s: %v", module.Source, destPath, err) + reader := sourcereader.Factory(mod.Source) + if err := reader.GetModule(mod.Source, fullPath); err != nil { + return fmt.Errorf("failed to get module from %s to %s: %v", mod.Source, fullPath, err) } /* Create module level files */ - writer := factory(module.Kind) + writer := factory(mod.Kind) writer.addNumModules(1) } } diff --git a/pkg/modulewriter/modulewriter_test.go b/pkg/modulewriter/modulewriter_test.go index ec891957ca..0fd7f73008 100644 --- a/pkg/modulewriter/modulewriter_test.go +++ b/pkg/modulewriter/modulewriter_test.go @@ -674,8 +674,9 @@ func (s *MySuite) TestWriteDeploymentGroup_PackerWriter(c *C) { } testPackerModule := config.Module{ - Kind: "packer", - ID: "testPackerModule", + Kind: "packer", + ID: "testPackerModule", + DeploymentSource: "testPackerModule", } testDeploymentGroup := config.DeploymentGroup{ Name: "packerGroup", diff --git a/pkg/modulewriter/packerwriter.go b/pkg/modulewriter/packerwriter.go index f5c42a0226..310ccc0f9a 100644 --- a/pkg/modulewriter/packerwriter.go +++ b/pkg/modulewriter/packerwriter.go @@ -80,7 +80,7 @@ func (w PackerWriter) writeDeploymentGroup( if err != nil { return err } - modPath := filepath.Join(groupPath, mod.ID) + modPath := filepath.Join(groupPath, mod.DeploymentSource) err = writePackerAutovars(ctySettings, modPath) if err != nil { return err diff --git a/pkg/modulewriter/tfwriter.go b/pkg/modulewriter/tfwriter.go index 72fcb55c8a..71cc82c173 100644 --- a/pkg/modulewriter/tfwriter.go +++ b/pkg/modulewriter/tfwriter.go @@ -30,7 +30,6 @@ import ( "github.com/zclconf/go-cty/cty" "hpc-toolkit/pkg/config" - "hpc-toolkit/pkg/sourcereader" ) const ( @@ -239,14 +238,7 @@ func writeMain( moduleBody := moduleBlock.Body() // Add source attribute - var moduleSource cty.Value - if sourcereader.IsGitPath(mod.Source) { - moduleSource = cty.StringVal(mod.Source) - } else { - moduleSource = cty.StringVal(fmt.Sprintf("./modules/%s", mod.ModuleName)) - } - - moduleBody.SetAttributeValue("source", moduleSource) + moduleBody.SetAttributeValue("source", cty.StringVal(mod.DeploymentSource)) // For each Setting for _, setting := range orderKeys(ctySettings) { From 97dbdbec9c5e810484ecc109ec542dd7cb373c3d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 24 Mar 2023 14:13:33 -0500 Subject: [PATCH 058/100] Resolve accidental destruction of startup-scripts When the startup-script module is used with VM instance templates that have create_before_destroy enabled (a common setting), a change to the script contents will result in the script being created, then destroyed. This commit resolves that by ensuring that all startup-script objects are created with unique, repeatable identifiers. This allows the new object to be created while destroying the old object with a different identifier. --- modules/scripts/startup-script/main.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/scripts/startup-script/main.tf b/modules/scripts/startup-script/main.tf index 1f265035e4..9e23e6c69a 100644 --- a/modules/scripts/startup-script/main.tf +++ b/modules/scripts/startup-script/main.tf @@ -64,8 +64,7 @@ locals { stdlib = join("", local.stdlib_list) runners_map = { for runner in local.runners : - basename(runner["destination"]) - => { + basename(runner["destination"]) => { content = lookup(runner, "content", null) source = lookup(runner, "source", null) } @@ -89,9 +88,9 @@ resource "google_storage_bucket" "configs_bucket" { resource "google_storage_bucket_object" "scripts" { # this writes all scripts exactly once into GCS for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}" - content = each.value["content"] - source = each.value["source"] + name = "${local.storage_folder_path_prefix}${each.key}-${try(md5(each.value.content), filemd5(each.value.source))}" + content = each.value.content + source = each.value.source bucket = local.storage_bucket_name timeouts { create = "10m" From 7f6c0280368852d45d01d81a89916d4ea4f55997 Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Thu, 23 Mar 2023 19:53:42 -0700 Subject: [PATCH 059/100] Adding Lustre example. --- examples/README.md | 24 +++++++++++++++++++++++ examples/lustre.yaml | 45 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 examples/lustre.yaml diff --git a/examples/README.md b/examples/README.md index a8338e4a36..c79f0893b0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -23,6 +23,7 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /" * [hpc-cluster-amd-slurmv5.yaml](#hpc-cluster-amd-slurmv5yaml-) ![community-badge] * [cloud-batch.yaml](#cloud-batchyaml-) ![core-badge] * [batch-mpi.yaml](#batch-mpiyaml-) ![core-badge] + * [lustre.yaml](#lustreyaml-) ![core-badge] * [spack-gromacs.yaml](#spack-gromacsyaml--) ![community-badge] ![experimental-badge] * [omnia-cluster.yaml](#omnia-clusteryaml--) ![community-badge] ![experimental-badge] * [hpc-cluster-small-sharedvpc.yaml](#hpc-cluster-small-sharedvpcyaml--) ![community-badge] ![experimental-badge] @@ -550,6 +551,29 @@ job. [cloud-batch.yaml]: ../examples/cloud-batch.yaml +### [lustre.yaml] ![core-badge] + +Creates a DDN EXAScaler lustre file-system that is mounted in two client instances. + +The [DDN Exascaler Lustre](../community/modules/file-system/DDN-EXAScaler/README.md) +file system is designed for high IO performance. It has a default capacity of ~10TiB and is mounted at `/lustre`. + +After the creation of the file-system and the client instances, the lustre drivers will be automatically installed and the mount-point configured on the VMs. This may take a few minutes after the VMs are created and can be verified by running: + +```sh +watch mount -t lustre +``` + +#### Quota Requirements for lustre.yaml + +For this example the following is needed in the selected region: + +* Compute Engine API: Persistent Disk SSD (GB): **~14TB: 3500GB MDT, 3500GB OST[0-2]** +* Compute Engine API: Persistent Disk Standard (GB): **~756GB: 20GB MDS, 276GB MGS, 3x20GB OSS, 2x200GB client-vms** +* Compute Engine API: N2 CPUs: **~116: 32 MDS, 32 MGS, 3x16 OSS, 2x2 client-vms** + +[lustre.yaml]: ./lustre.yaml + ### [batch-mpi.yaml] ![core-badge] This blueprint demonstrates how to use Spack to run a real MPI job on Batch. diff --git a/examples/lustre.yaml b/examples/lustre.yaml new file mode 100644 index 0000000000..e456242387 --- /dev/null +++ b/examples/lustre.yaml @@ -0,0 +1,45 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +blueprint_name: small-lustre-example + +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: small-lustre + region: us-central1 + zone: us-central1-c + +# Documentation for each of the modules used below can be found at +# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md + +deployment_groups: +- group: primary + modules: + - id: network1 + source: modules/network/pre-existing-vpc + + - id: lustre + source: community/modules/file-system/DDN-EXAScaler + use: [network1] + settings: + local_mount: /lustre + + - source: modules/compute/vm-instance + id: compute_instances + use: [network1, lustre] + settings: + name_prefix: client-vm + instance_count: 2 + machine_type: n2-standard-2 From 8722b6d66d62640c1820b55fba179b202cbb0bfd Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Fri, 24 Mar 2023 13:56:28 -0700 Subject: [PATCH 060/100] Updating image in the slurm-gcp-v5 Ubuntu example --- community/examples/slurm-gcp-v5-ubuntu2004.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/community/examples/slurm-gcp-v5-ubuntu2004.yaml b/community/examples/slurm-gcp-v5-ubuntu2004.yaml index f2891f3b78..69f16e54a4 100644 --- a/community/examples/slurm-gcp-v5-ubuntu2004.yaml +++ b/community/examples/slurm-gcp-v5-ubuntu2004.yaml @@ -22,7 +22,9 @@ vars: region: us-central1 zone: us-central1-c instance_image: - family: schedmd-v5-slurm-22-05-6-ubuntu-2004-lts + # Please refer to the following link for the latest images: + # https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems + family: schedmd-v5-slurm-22-05-8-ubuntu-2004-lts project: projects/schedmd-slurm-public/global/images/family From 97079a6faace3f9c709f04a6bd71b95daf790185 Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Fri, 24 Mar 2023 14:25:27 -0700 Subject: [PATCH 061/100] Show core examples first --- examples/README.md | 505 +++++++++++++++++++++++---------------------- 1 file changed, 253 insertions(+), 252 deletions(-) diff --git a/examples/README.md b/examples/README.md index c79f0893b0..40cc0b3283 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,23 +13,23 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /" * [Blueprint Descriptions](#blueprint-descriptions) * [hpc-cluster-small.yaml](#hpc-cluster-smallyaml-) ![core-badge] * [hpc-cluster-high-io.yaml](#hpc-cluster-high-ioyaml-) ![core-badge] + * [image-builder.yaml](#image-builderyaml-) ![core-badge] + * [cloud-batch.yaml](#cloud-batchyaml-) ![core-badge] + * [batch-mpi.yaml](#batch-mpiyaml-) ![core-badge] + * [lustre.yaml](#lustreyaml-) ![core-badge] * [slurm-gcp-v5-hpc-centos7.yaml](#slurm-gcp-v5-hpc-centos7yaml-) ![community-badge] * [slurm-gcp-v5-ubuntu2004.yaml](#slurm-gcp-v5-ubuntu2004yaml-) ![community-badge] * [slurm-gcp-v5-high-io.yaml](#slurm-gcp-v5-high-ioyaml-) ![community-badge] - * [image-builder.yaml](#image-builderyaml-) ![core-badge] * [hpc-cluster-intel-select.yaml](#hpc-cluster-intel-selectyaml-) ![community-badge] * [daos-cluster.yaml](#daos-clusteryaml-) ![community-badge] * [daos-slurm.yaml](#daos-slurmyaml-) ![community-badge] * [hpc-cluster-amd-slurmv5.yaml](#hpc-cluster-amd-slurmv5yaml-) ![community-badge] - * [cloud-batch.yaml](#cloud-batchyaml-) ![core-badge] - * [batch-mpi.yaml](#batch-mpiyaml-) ![core-badge] - * [lustre.yaml](#lustreyaml-) ![core-badge] + * [quantum-circuit-simulator.yaml](#quantum-circuit-simulatoryaml-) ![community-badge] * [spack-gromacs.yaml](#spack-gromacsyaml--) ![community-badge] ![experimental-badge] * [omnia-cluster.yaml](#omnia-clusteryaml--) ![community-badge] ![experimental-badge] * [hpc-cluster-small-sharedvpc.yaml](#hpc-cluster-small-sharedvpcyaml--) ![community-badge] ![experimental-badge] * [hpc-cluster-localssd.yaml](#hpc-cluster-localssdyaml--) ![community-badge] ![experimental-badge] * [htcondor-pool.yaml](#htcondor-poolyaml--) ![community-badge] ![experimental-badge] - * [quantum-circuit-simulator.yaml](#quantum-circuit-simulatoryaml-) ![community-badge] * [starccm-tutorial.yaml](#starccm-tutorialyaml--) ![community-badge] ![experimental-badge] * [fluent-tutorial.yaml](#fluent-tutorialyaml--) ![community-badge] ![experimental-badge] * [Blueprint Schema](#blueprint-schema) @@ -41,6 +41,7 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /" * [Variables](#variables) * [Blueprint Variables](#blueprint-variables) * [Literal Variables](#literal-variables) + * [Escape Variables](#escape-variables) ## Instructions @@ -184,170 +185,6 @@ For this example the following is needed in the selected region: [hpc-cluster-high-io.yaml]: ./hpc-cluster-high-io.yaml -### [slurm-gcp-v5-hpc-centos7.yaml] ![community-badge] - -This example creates an HPC cluster similar to the one created by -[hpc-cluster-small.yaml], but uses modules built from version 5 of -[slurm-gcp]. - -The cluster will support 2 partitions named `debug` and `compute`. -The `debug` partition is the default partition and runs on smaller -`n2-standard-2` nodes. The `compute` partition is not default and requires -specifying in the `srun` command via the `--partition` flag. The `compute` -partition runs on compute optimized nodes of type `cs-standard-60`. The -`compute` partition may require additional quota before using. - -#### Quota Requirements for slurm-gcp-v5-hpc-centos7.yaml - -For this example the following is needed in the selected region: - -* Cloud Filestore API: Basic HDD (Standard) capacity (GB): **1,024 GB** -* Compute Engine API: Persistent Disk SSD (GB): **~50 GB** -* Compute Engine API: Persistent Disk Standard (GB): **~50 GB static + 50 - GB/node** up to 1,250 GB -* Compute Engine API: N2 CPUs: **12** -* Compute Engine API: C2 CPUs: **4** for controller node and **60/node** active - in `compute` partition up to 1,204 -* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only - needed for `compute` partition_ -* Compute Engine API: Resource policies: **one for each job in parallel** - - _only needed for `compute` partition_ - -[slurm-gcp-v5-hpc-centos7.yaml]: ../community/examples/slurm-gcp-v5-hpc-centos7.yaml -[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.2.0 - -### [slurm-gcp-v5-ubuntu2004.yaml] ![community-badge] - -Similar to the previous example, but using Ubuntu 20.04 instead of CentOS 7. -[Other operating systems] are supported by SchedMD for the the Slurm on GCP project and images are listed [here](https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#published-image-family). Only the examples listed in this page been tested by the Cloud HPC Toolkit team. - -This example creates an HPC cluster similar to the one created by -[hpc-cluster-small.yaml], but uses modules built from version 5 of -[slurm-gcp] and Ubuntu. - -The cluster will support 2 partitions named `debug` and `compute`. -The `debug` partition is the default partition and runs on smaller -`n2-standard-2` nodes. The `compute` partition is not default and requires -specifying in the `srun` command via the `--partition` flag. The `compute` -partition runs on compute optimized nodes of type `cs-standard-60`. The -`compute` partition may require additional quota before using. - -[Other operating systems]: https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems -[slurm-gcp-v5-ubuntu2004.yaml]: ../community/examples/slurm-gcp-v5-ubuntu2004.yaml - -#### Quota Requirements for slurm-gcp-v5-ubuntu2004.yaml - -For this example the following is needed in the selected region: - -* Cloud Filestore API: Basic HDD (Standard) capacity (GB): **1,024 GB** -* Compute Engine API: Persistent Disk SSD (GB): **~50 GB** -* Compute Engine API: Persistent Disk Standard (GB): **~50 GB static + 50 - GB/node** up to 1,250 GB -* Compute Engine API: N2 CPUs: **12** -* Compute Engine API: C2 CPUs: **4** for controller node and **60/node** active - in `compute` partition up to 1,204 -* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only - needed for `compute` partition_ -* Compute Engine API: Resource policies: **one for each job in parallel** - - _only needed for `compute` partition_ - -### [slurm-gcp-v5-high-io.yaml] ![community-badge] - -This example uses [Slurm on GCP][slurm-gcp] version 5.x modules to replicate the -[hpc-cluster-high-io.yaml] core example. With version 5, additional features are -available and utilized in this example: - -* node groups are used to allow multiple machine types in a single partition, - differentiated by node names. -* Active cluster reconfiguration is on by default. When updating a partition or - cluster configuration, the overwrite option (`-w`) can be used and upon - re-applying the deployment, the changes will become active without having to - destroy and recreate the cluster. - -This blueprint will create a cluster with the following storage tiers: - -* The homefs mounted at `/home` is a default "BASIC_HDD" tier filestore with - 1 TiB of capacity -* The projectsfs is mounted at `/projects` and is a high scale SSD filestore - instance with 10TiB of capacity. -* The scratchfs is mounted at `/scratch` and is a - [DDN Exascaler Lustre](../community/modules/file-system/DDN-EXAScaler/README.md) - file system designed for high IO performance. The capacity is ~10TiB. - -The cluster will support 2 partitions: - -* `lowcost` - * Includes two node groups, `n2s2` of machine type `n2-standard-2` and `n2s4` - of machine type `n2-standard-4`. - * Default partition. - * Designed to run with lower cost nodes and within a typical project's default - quota. -* `compute` - * Includes two node groups, `c2s60` of machine type `c2-standard-60` and - `c2s30` of machine type `c2-standard-30`. - * Can be used by setting the `--partition` option in `srun` to `compute`. - * Designed for performance, but may require additional quota before using. - -[slurm-gcp-v5-high-io.yaml]: ../community/examples/slurm-gcp-v5-high-io.yaml - -#### Usage of Node Groups -This example defines partitions with more than one node group each. For more -information on node groups and why they are used, see the documentation in the -[schedmd-slurm-gcp-v5-node-group] module documentation. Some reference commands -are listed here for specifying not only the partition, but also the correct node -group when executing a Slurm command on a cluster generated by this blueprint. - -Partition: compute; Node Group: c2s30; Machine Type: c2-standard-30 - -```bash -srun -N 4 -p compute -w highioslur-compute-c2s30-[0-3] hostname -``` - -Partition: compute; Node Group: c2s60; Machine Type: c2-standard-60 - -```bash -srun -N 4 -p compute --mincpus=30 hostname -``` - -Partition: lowcost; Node Group: n2s2; Machine Type: n2-standard-2 - -```bash -srun -N 4 -w highioslur-lowcost-n2s2-[0-3] hostname -``` - -Partition: lowcost; Node Group: n2s4; Machine Type: n2-standard-4 - -```bash -srun -N 4 --mincpus=2 hostname -``` - -[schedmd-slurm-gcp-v5-node-group]: ../community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md - -#### Quota Requirements for slurm-gcp-v5-high-io.yaml - -For this example the following is needed in the selected region: - -* Cloud Filestore API: Basic HDD (Standard) capacity (GB) per region: **1,024 GB** -* Cloud Filestore API: High Scale SSD capacity (GB) per region: **10,240 GiB** - _min - quota request is 61,440 GiB_ -* Compute Engine API: Persistent Disk SSD (GB): **~14,050 GB** -* Compute Engine API: Persistent Disk Standard (GB): **~396 GB static + 20 - GB/node** up to 4596 GB -* Compute Engine API: N2 CPUs: - * **4** for the login node - * **2** per node for active nodes in the `n2s2` group, maximum 20. - * **4** per node for active nodes in the `n2s4` group, maximum 40. - * Maximum possible: **64** -* Compute Engine API: C2 CPUs: - * **8** for controller node - * **60** per node for active nodes in the `c2s60` group, maximum 12,000. - * **30** per node for active nodes in the `c2s30` group, maximum 6,000. - * Maximum possible: **18,008** -* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only - needed for `compute` partition_ -* Compute Engine API: Resource policies: **one for each job in parallel** - - _only needed for `compute` partition_ - ### [image-builder.yaml] ![core-badge] This Blueprint uses the [Packer template module][pkr] to create custom VM images @@ -498,45 +335,6 @@ file that was added during image build: Hello World ``` -### [hpc-cluster-intel-select.yaml] ![community-badge] - -This example provisions a Slurm cluster automating the [steps to comply to the -Intel Select Solutions for Simulation & Modeling Criteria][intelselect]. It is -more extensively discussed in a dedicated [README for Intel -examples][intel-examples-readme]. - -[hpc-cluster-intel-select.yaml]: ../community/examples/intel/hpc-cluster-intel-select.yaml -[intel-examples-readme]: ../community/examples/intel/README.md -[intelselect]: https://cloud.google.com/compute/docs/instances/create-intel-select-solution-hpc-clusters - -### [daos-cluster.yaml] ![community-badge] - -This example provisions a DAOS cluster with [managed instance groups][migs] for the servers and for clients. It is more extensively discussed in a dedicated [README for Intel -examples][intel-examples-readme]. - -[daos-cluster.yaml]: ../community/examples/intel/daos-cluster.yaml -[migs]: https://cloud.google.com/compute/docs/instance-groups - -### [daos-slurm.yaml] ![community-badge] - -This example provisions DAOS servers and a Slurm cluster. It is -more extensively discussed in a dedicated [README for Intel -examples][intel-examples-readme]. - -[daos-slurm.yaml]: ../community/examples/intel/daos-slurm.yaml - -### [hpc-cluster-amd-slurmv5.yaml] ![community-badge] - -This example provisions a Slurm cluster using AMD VM machine types. It -automates the initial setup of Spack, including a script that can be used to -install the AMD Optimizing C/C++ Compiler ([AOCC]) and compile OpenMPI with -AOCC. It is more extensively discussed in a dedicated [README for AMD -examples][amd-examples-readme]. - -[hpc-cluster-amd-slurmv5.yaml]: ../community/examples/AMD/hpc-cluster-amd-slurmv5.yaml -[AOCC]: https://developer.amd.com/amd-aocc/ -[amd-examples-readme]: ../community/examples/AMD/README.md - ### [cloud-batch.yaml] ![core-badge] This example demonstrates how to use the HPC Toolkit to set up a Google Cloud Batch job @@ -551,29 +349,6 @@ job. [cloud-batch.yaml]: ../examples/cloud-batch.yaml -### [lustre.yaml] ![core-badge] - -Creates a DDN EXAScaler lustre file-system that is mounted in two client instances. - -The [DDN Exascaler Lustre](../community/modules/file-system/DDN-EXAScaler/README.md) -file system is designed for high IO performance. It has a default capacity of ~10TiB and is mounted at `/lustre`. - -After the creation of the file-system and the client instances, the lustre drivers will be automatically installed and the mount-point configured on the VMs. This may take a few minutes after the VMs are created and can be verified by running: - -```sh -watch mount -t lustre -``` - -#### Quota Requirements for lustre.yaml - -For this example the following is needed in the selected region: - -* Compute Engine API: Persistent Disk SSD (GB): **~14TB: 3500GB MDT, 3500GB OST[0-2]** -* Compute Engine API: Persistent Disk Standard (GB): **~756GB: 20GB MDS, 276GB MGS, 3x20GB OSS, 2x200GB client-vms** -* Compute Engine API: N2 CPUs: **~116: 32 MDS, 32 MGS, 3x16 OSS, 2x2 client-vms** - -[lustre.yaml]: ./lustre.yaml - ### [batch-mpi.yaml] ![core-badge] This blueprint demonstrates how to use Spack to run a real MPI job on Batch. @@ -630,6 +405,253 @@ The blueprint contains the following: [batch-mpi.yaml]: ../examples/batch-mpi.yaml +### [lustre.yaml] ![core-badge] + +Creates a DDN EXAScaler lustre file-system that is mounted in two client instances. + +The [DDN Exascaler Lustre](../community/modules/file-system/DDN-EXAScaler/README.md) +file system is designed for high IO performance. It has a default capacity of ~10TiB and is mounted at `/lustre`. + +After the creation of the file-system and the client instances, the lustre drivers will be automatically installed and the mount-point configured on the VMs. This may take a few minutes after the VMs are created and can be verified by running: + +```sh +watch mount -t lustre +``` + +#### Quota Requirements for lustre.yaml + +For this example the following is needed in the selected region: + +* Compute Engine API: Persistent Disk SSD (GB): **~14TB: 3500GB MDT, 3500GB OST[0-2]** +* Compute Engine API: Persistent Disk Standard (GB): **~756GB: 20GB MDS, 276GB MGS, 3x20GB OSS, 2x200GB client-vms** +* Compute Engine API: N2 CPUs: **~116: 32 MDS, 32 MGS, 3x16 OSS, 2x2 client-vms** + +[lustre.yaml]: ./lustre.yaml + +### [slurm-gcp-v5-hpc-centos7.yaml] ![community-badge] + +This example creates an HPC cluster similar to the one created by +[hpc-cluster-small.yaml], but uses modules built from version 5 of +[slurm-gcp]. + +The cluster will support 2 partitions named `debug` and `compute`. +The `debug` partition is the default partition and runs on smaller +`n2-standard-2` nodes. The `compute` partition is not default and requires +specifying in the `srun` command via the `--partition` flag. The `compute` +partition runs on compute optimized nodes of type `cs-standard-60`. The +`compute` partition may require additional quota before using. + +#### Quota Requirements for slurm-gcp-v5-hpc-centos7.yaml + +For this example the following is needed in the selected region: + +* Cloud Filestore API: Basic HDD (Standard) capacity (GB): **1,024 GB** +* Compute Engine API: Persistent Disk SSD (GB): **~50 GB** +* Compute Engine API: Persistent Disk Standard (GB): **~50 GB static + 50 + GB/node** up to 1,250 GB +* Compute Engine API: N2 CPUs: **12** +* Compute Engine API: C2 CPUs: **4** for controller node and **60/node** active + in `compute` partition up to 1,204 +* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only + needed for `compute` partition_ +* Compute Engine API: Resource policies: **one for each job in parallel** - + _only needed for `compute` partition_ + +[slurm-gcp-v5-hpc-centos7.yaml]: ../community/examples/slurm-gcp-v5-hpc-centos7.yaml +[slurm-gcp]: https://github.com/SchedMD/slurm-gcp/tree/5.2.0 + +### [slurm-gcp-v5-ubuntu2004.yaml] ![community-badge] + +Similar to the previous example, but using Ubuntu 20.04 instead of CentOS 7. +[Other operating systems] are supported by SchedMD for the the Slurm on GCP project and images are listed [here](https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#published-image-family). Only the examples listed in this page been tested by the Cloud HPC Toolkit team. + +This example creates an HPC cluster similar to the one created by +[hpc-cluster-small.yaml], but uses modules built from version 5 of +[slurm-gcp] and Ubuntu. + +The cluster will support 2 partitions named `debug` and `compute`. +The `debug` partition is the default partition and runs on smaller +`n2-standard-2` nodes. The `compute` partition is not default and requires +specifying in the `srun` command via the `--partition` flag. The `compute` +partition runs on compute optimized nodes of type `cs-standard-60`. The +`compute` partition may require additional quota before using. + +[Other operating systems]: https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems +[slurm-gcp-v5-ubuntu2004.yaml]: ../community/examples/slurm-gcp-v5-ubuntu2004.yaml + +#### Quota Requirements for slurm-gcp-v5-ubuntu2004.yaml + +For this example the following is needed in the selected region: + +* Cloud Filestore API: Basic HDD (Standard) capacity (GB): **1,024 GB** +* Compute Engine API: Persistent Disk SSD (GB): **~50 GB** +* Compute Engine API: Persistent Disk Standard (GB): **~50 GB static + 50 + GB/node** up to 1,250 GB +* Compute Engine API: N2 CPUs: **12** +* Compute Engine API: C2 CPUs: **4** for controller node and **60/node** active + in `compute` partition up to 1,204 +* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only + needed for `compute` partition_ +* Compute Engine API: Resource policies: **one for each job in parallel** - + _only needed for `compute` partition_ + +### [slurm-gcp-v5-high-io.yaml] ![community-badge] + +This example uses [Slurm on GCP][slurm-gcp] version 5.x modules to replicate the +[hpc-cluster-high-io.yaml] core example. With version 5, additional features are +available and utilized in this example: + +* node groups are used to allow multiple machine types in a single partition, + differentiated by node names. +* Active cluster reconfiguration is on by default. When updating a partition or + cluster configuration, the overwrite option (`-w`) can be used and upon + re-applying the deployment, the changes will become active without having to + destroy and recreate the cluster. + +This blueprint will create a cluster with the following storage tiers: + +* The homefs mounted at `/home` is a default "BASIC_HDD" tier filestore with + 1 TiB of capacity +* The projectsfs is mounted at `/projects` and is a high scale SSD filestore + instance with 10TiB of capacity. +* The scratchfs is mounted at `/scratch` and is a + [DDN Exascaler Lustre](../community/modules/file-system/DDN-EXAScaler/README.md) + file system designed for high IO performance. The capacity is ~10TiB. + +The cluster will support 2 partitions: + +* `lowcost` + * Includes two node groups, `n2s2` of machine type `n2-standard-2` and `n2s4` + of machine type `n2-standard-4`. + * Default partition. + * Designed to run with lower cost nodes and within a typical project's default + quota. +* `compute` + * Includes two node groups, `c2s60` of machine type `c2-standard-60` and + `c2s30` of machine type `c2-standard-30`. + * Can be used by setting the `--partition` option in `srun` to `compute`. + * Designed for performance, but may require additional quota before using. + +[slurm-gcp-v5-high-io.yaml]: ../community/examples/slurm-gcp-v5-high-io.yaml + +#### Usage of Node Groups +This example defines partitions with more than one node group each. For more +information on node groups and why they are used, see the documentation in the +[schedmd-slurm-gcp-v5-node-group] module documentation. Some reference commands +are listed here for specifying not only the partition, but also the correct node +group when executing a Slurm command on a cluster generated by this blueprint. + +Partition: compute; Node Group: c2s30; Machine Type: c2-standard-30 + +```bash +srun -N 4 -p compute -w highioslur-compute-c2s30-[0-3] hostname +``` + +Partition: compute; Node Group: c2s60; Machine Type: c2-standard-60 + +```bash +srun -N 4 -p compute --mincpus=30 hostname +``` + +Partition: lowcost; Node Group: n2s2; Machine Type: n2-standard-2 + +```bash +srun -N 4 -w highioslur-lowcost-n2s2-[0-3] hostname +``` + +Partition: lowcost; Node Group: n2s4; Machine Type: n2-standard-4 + +```bash +srun -N 4 --mincpus=2 hostname +``` + +[schedmd-slurm-gcp-v5-node-group]: ../community/modules/compute/schedmd-slurm-gcp-v5-node-group/README.md + +#### Quota Requirements for slurm-gcp-v5-high-io.yaml + +For this example the following is needed in the selected region: + +* Cloud Filestore API: Basic HDD (Standard) capacity (GB) per region: **1,024 GB** +* Cloud Filestore API: High Scale SSD capacity (GB) per region: **10,240 GiB** - _min + quota request is 61,440 GiB_ +* Compute Engine API: Persistent Disk SSD (GB): **~14,050 GB** +* Compute Engine API: Persistent Disk Standard (GB): **~396 GB static + 20 + GB/node** up to 4596 GB +* Compute Engine API: N2 CPUs: + * **4** for the login node + * **2** per node for active nodes in the `n2s2` group, maximum 20. + * **4** per node for active nodes in the `n2s4` group, maximum 40. + * Maximum possible: **64** +* Compute Engine API: C2 CPUs: + * **8** for controller node + * **60** per node for active nodes in the `c2s60` group, maximum 12,000. + * **30** per node for active nodes in the `c2s30` group, maximum 6,000. + * Maximum possible: **18,008** +* Compute Engine API: Affinity Groups: **one for each job in parallel** - _only + needed for `compute` partition_ +* Compute Engine API: Resource policies: **one for each job in parallel** - + _only needed for `compute` partition_ + +### [hpc-cluster-intel-select.yaml] ![community-badge] + +This example provisions a Slurm cluster automating the [steps to comply to the +Intel Select Solutions for Simulation & Modeling Criteria][intelselect]. It is +more extensively discussed in a dedicated [README for Intel +examples][intel-examples-readme]. + +[hpc-cluster-intel-select.yaml]: ../community/examples/intel/hpc-cluster-intel-select.yaml +[intel-examples-readme]: ../community/examples/intel/README.md +[intelselect]: https://cloud.google.com/compute/docs/instances/create-intel-select-solution-hpc-clusters + +### [daos-cluster.yaml] ![community-badge] + +This example provisions a DAOS cluster with [managed instance groups][migs] for the servers and for clients. It is more extensively discussed in a dedicated [README for Intel +examples][intel-examples-readme]. + +[daos-cluster.yaml]: ../community/examples/intel/daos-cluster.yaml +[migs]: https://cloud.google.com/compute/docs/instance-groups + +### [daos-slurm.yaml] ![community-badge] + +This example provisions DAOS servers and a Slurm cluster. It is +more extensively discussed in a dedicated [README for Intel +examples][intel-examples-readme]. + +[daos-slurm.yaml]: ../community/examples/intel/daos-slurm.yaml + +### [hpc-cluster-amd-slurmv5.yaml] ![community-badge] + +This example provisions a Slurm cluster using AMD VM machine types. It +automates the initial setup of Spack, including a script that can be used to +install the AMD Optimizing C/C++ Compiler ([AOCC]) and compile OpenMPI with +AOCC. It is more extensively discussed in a dedicated [README for AMD +examples][amd-examples-readme]. + +[hpc-cluster-amd-slurmv5.yaml]: ../community/examples/AMD/hpc-cluster-amd-slurmv5.yaml +[AOCC]: https://developer.amd.com/amd-aocc/ +[amd-examples-readme]: ../community/examples/AMD/README.md + +### [quantum-circuit-simulator.yaml] ![community-badge] + +This blueprint provisions an [A2 series VM with NVIDIA A100 GPU accelerator][a2] +and compiles [qsim], a [Google Quantum AI][gqai]-developed tool that simulates +quantum circuits using CPUs and GPUs. The installation of qsim, the [CUDA +Toolkit][cudatk], and the [cuQuantum SDK][cqsdk] is fully automated but takes a +significant time (approx. 20 minutes). Once complete, a qsim example can be run +by connecting to the VM by SSH and running + +```shell +conda activate qsim +python /var/tmp/qsim-example.py +``` + +[gqai]: https://quantumai.google/ +[quantum-circuit-simulator.yaml]: ../community/examples/quantum-circuit-simulator.yaml +[a2]: https://cloud.google.com/compute/docs/gpus#a100-gpus +[qsim]: https://quantumai.google/qsim +[cqsdk]: https://developer.nvidia.com/cuquantum-sdk +[cudatk]: https://developer.nvidia.com/cuda-toolkit + ### [spack-gromacs.yaml] ![community-badge] ![experimental-badge] Spack is an HPC software package manager. This example creates a small Slurm @@ -728,27 +750,6 @@ walks through the use of this blueprint. [htcondor-pool.yaml]: ../community/examples/htcondor-pool.yaml [hpcvmimage]: https://cloud.google.com/compute/docs/instances/create-hpc-vm -### [quantum-circuit-simulator.yaml] ![community-badge] - -This blueprint provisions an [A2 series VM with NVIDIA A100 GPU accelerator][a2] -and compiles [qsim], a [Google Quantum AI][gqai]-developed tool that simulates -quantum circuits using CPUs and GPUs. The installation of qsim, the [CUDA -Toolkit][cudatk], and the [cuQuantum SDK][cqsdk] is fully automated but takes a -significant time (approx. 20 minutes). Once complete, a qsim example can be run -by connecting to the VM by SSH and running - -```shell -conda activate qsim -python /var/tmp/qsim-example.py -``` - -[gqai]: https://quantumai.google/ -[quantum-circuit-simulator.yaml]: ../community/examples/quantum-circuit-simulator.yaml -[a2]: https://cloud.google.com/compute/docs/gpus#a100-gpus -[qsim]: https://quantumai.google/qsim -[cqsdk]: https://developer.nvidia.com/cuquantum-sdk -[cudatk]: https://developer.nvidia.com/cuda-toolkit - ### [starccm-tutorial.yaml] ![community-badge] ![experimental-badge] This blueprint provisions a simple cluster for use with a Simcenter StarCCM+ From 4bc298fb22032188a32c8714b3273c399dd4b4f1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 11:00:48 +0000 Subject: [PATCH 062/100] Bump cloud.google.com/go/compute from 1.18.0 to 1.19.0 Bumps [cloud.google.com/go/compute](https://github.com/googleapis/google-cloud-go) from 1.18.0 to 1.19.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/documentai/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.18.0...pubsub/v1.19.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/compute dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index af03cce168..c052bd5965 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module hpc-toolkit go 1.18 require ( - cloud.google.com/go/compute v1.18.0 + cloud.google.com/go/compute v1.19.0 cloud.google.com/go/storage v1.28.1 // indirect github.com/go-git/go-git/v5 v5.6.1 github.com/hashicorp/go-getter v1.7.1 @@ -16,7 +16,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/zclconf/go-cty v1.13.1 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 + google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 9c305a87ef..236c9fda10 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -955,8 +955,8 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 h1:khxVcsk/FhnzxMKOyD+TDGwjbEOpcPuIpmafPGFmhMA= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 80b9cd468277f60e790adc6518169258c34da65d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 27 Mar 2023 08:59:47 -0500 Subject: [PATCH 063/100] Address feedback in #1085 --- modules/scripts/startup-script/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/scripts/startup-script/main.tf b/modules/scripts/startup-script/main.tf index 9e23e6c69a..9b93cde646 100644 --- a/modules/scripts/startup-script/main.tf +++ b/modules/scripts/startup-script/main.tf @@ -88,7 +88,7 @@ resource "google_storage_bucket" "configs_bucket" { resource "google_storage_bucket_object" "scripts" { # this writes all scripts exactly once into GCS for_each = local.runners_map - name = "${local.storage_folder_path_prefix}${each.key}-${try(md5(each.value.content), filemd5(each.value.source))}" + name = "${local.storage_folder_path_prefix}${each.key}-${substr(try(md5(each.value.content), filemd5(each.value.source)), 0, 4)}" content = each.value.content source = each.value.source bucket = local.storage_bucket_name From 6c6b9e0a5e18fc1c1d7fe762680db7fd4f3901a5 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 27 Mar 2023 11:27:01 -0500 Subject: [PATCH 064/100] Use latest update to startup-script module --- community/modules/compute/pbspro-execution/README.md | 2 +- community/modules/compute/pbspro-execution/main.tf | 2 +- .../modules/remote-desktop/chrome-remote-desktop/README.md | 2 +- community/modules/remote-desktop/chrome-remote-desktop/main.tf | 2 +- community/modules/scheduler/pbspro-client/README.md | 2 +- community/modules/scheduler/pbspro-client/main.tf | 2 +- community/modules/scheduler/pbspro-server/README.md | 2 +- community/modules/scheduler/pbspro-server/main.tf | 2 +- modules/compute/vm-instance/README.md | 2 +- modules/compute/vm-instance/startup_from_network_storage.tf | 2 +- modules/scheduler/batch-job-template/README.md | 2 +- .../batch-job-template/startup_from_network_storage.tf | 2 +- modules/scheduler/batch-login-node/README.md | 2 +- modules/scheduler/batch-login-node/main.tf | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/community/modules/compute/pbspro-execution/README.md b/community/modules/compute/pbspro-execution/README.md index 96107e8a05..5a8630e130 100644 --- a/community/modules/compute/pbspro-execution/README.md +++ b/community/modules/compute/pbspro-execution/README.md @@ -74,7 +74,7 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [execution\_startup\_script](#module\_execution\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [execution\_startup\_script](#module\_execution\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | | [pbs\_execution](#module\_pbs\_execution) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/compute/vm-instance | 264e99c | | [pbs\_install](#module\_pbs\_install) | github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/pbspro-install | be2673b | diff --git a/community/modules/compute/pbspro-execution/main.tf b/community/modules/compute/pbspro-execution/main.tf index e75a53f938..27fbd23501 100644 --- a/community/modules/compute/pbspro-execution/main.tf +++ b/community/modules/compute/pbspro-execution/main.tf @@ -48,7 +48,7 @@ module "pbs_install" { } module "execution_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" deployment_name = var.deployment_name project_id = var.project_id diff --git a/community/modules/remote-desktop/chrome-remote-desktop/README.md b/community/modules/remote-desktop/chrome-remote-desktop/README.md index 448fc6c95f..6ddb930110 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/README.md +++ b/community/modules/remote-desktop/chrome-remote-desktop/README.md @@ -63,7 +63,7 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 1b1cdb0 | +| [client\_startup\_script](#module\_client\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | | [instances](#module\_instances) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/compute/vm-instance | 264e99c | ## Resources diff --git a/community/modules/remote-desktop/chrome-remote-desktop/main.tf b/community/modules/remote-desktop/chrome-remote-desktop/main.tf index 15176c6067..f523830e43 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/main.tf +++ b/community/modules/remote-desktop/chrome-remote-desktop/main.tf @@ -51,7 +51,7 @@ locals { } module "client_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=1b1cdb0" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" deployment_name = var.deployment_name project_id = var.project_id diff --git a/community/modules/scheduler/pbspro-client/README.md b/community/modules/scheduler/pbspro-client/README.md index dcde7bfd6b..4022bef755 100644 --- a/community/modules/scheduler/pbspro-client/README.md +++ b/community/modules/scheduler/pbspro-client/README.md @@ -74,7 +74,7 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [client\_startup\_script](#module\_client\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [client\_startup\_script](#module\_client\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | | [pbs\_client](#module\_pbs\_client) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/compute/vm-instance | 264e99c | | [pbs\_install](#module\_pbs\_install) | github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/pbspro-install | be2673b | diff --git a/community/modules/scheduler/pbspro-client/main.tf b/community/modules/scheduler/pbspro-client/main.tf index f1564348a2..0565b610c9 100644 --- a/community/modules/scheduler/pbspro-client/main.tf +++ b/community/modules/scheduler/pbspro-client/main.tf @@ -38,7 +38,7 @@ module "pbs_install" { } module "client_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" deployment_name = var.deployment_name project_id = var.project_id diff --git a/community/modules/scheduler/pbspro-server/README.md b/community/modules/scheduler/pbspro-server/README.md index dbfd91b8bb..fa36f3334a 100644 --- a/community/modules/scheduler/pbspro-server/README.md +++ b/community/modules/scheduler/pbspro-server/README.md @@ -72,7 +72,7 @@ No providers. | [pbs\_install](#module\_pbs\_install) | github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/pbspro-install | be2673b | | [pbs\_qmgr](#module\_pbs\_qmgr) | github.com/GoogleCloudPlatform/hpc-toolkit//community/modules/scripts/pbspro-qmgr | be2673b | | [pbs\_server](#module\_pbs\_server) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/compute/vm-instance | 264e99c | -| [server\_startup\_script](#module\_server\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [server\_startup\_script](#module\_server\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | ## Resources diff --git a/community/modules/scheduler/pbspro-server/main.tf b/community/modules/scheduler/pbspro-server/main.tf index 2c2422194e..abc5f8d269 100644 --- a/community/modules/scheduler/pbspro-server/main.tf +++ b/community/modules/scheduler/pbspro-server/main.tf @@ -50,7 +50,7 @@ module "pbs_qmgr" { } module "server_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" deployment_name = var.deployment_name project_id = var.project_id diff --git a/modules/compute/vm-instance/README.md b/modules/compute/vm-instance/README.md index 891af62451..62c8869662 100644 --- a/modules/compute/vm-instance/README.md +++ b/modules/compute/vm-instance/README.md @@ -158,7 +158,7 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | ## Resources diff --git a/modules/compute/vm-instance/startup_from_network_storage.tf b/modules/compute/vm-instance/startup_from_network_storage.tf index 823c060ffb..752d9ee06c 100644 --- a/modules/compute/vm-instance/startup_from_network_storage.tf +++ b/modules/compute/vm-instance/startup_from_network_storage.tf @@ -55,7 +55,7 @@ locals { } module "netstorage_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" labels = var.labels project_id = var.project_id diff --git a/modules/scheduler/batch-job-template/README.md b/modules/scheduler/batch-job-template/README.md index a6496cb7a8..a1f57778c7 100644 --- a/modules/scheduler/batch-job-template/README.md +++ b/modules/scheduler/batch-job-template/README.md @@ -135,7 +135,7 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| | [instance\_template](#module\_instance\_template) | terraform-google-modules/vm/google//modules/instance_template | ~> 8.0 | -| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [netstorage\_startup\_script](#module\_netstorage\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | ## Resources diff --git a/modules/scheduler/batch-job-template/startup_from_network_storage.tf b/modules/scheduler/batch-job-template/startup_from_network_storage.tf index 823c060ffb..752d9ee06c 100644 --- a/modules/scheduler/batch-job-template/startup_from_network_storage.tf +++ b/modules/scheduler/batch-job-template/startup_from_network_storage.tf @@ -55,7 +55,7 @@ locals { } module "netstorage_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" labels = var.labels project_id = var.project_id diff --git a/modules/scheduler/batch-login-node/README.md b/modules/scheduler/batch-login-node/README.md index f483334008..122d574350 100644 --- a/modules/scheduler/batch-login-node/README.md +++ b/modules/scheduler/batch-login-node/README.md @@ -89,7 +89,7 @@ limitations under the License. | Name | Source | Version | |------|--------|---------| -| [login\_startup\_script](#module\_login\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 64bc890 | +| [login\_startup\_script](#module\_login\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | 34bb7250 | ## Resources diff --git a/modules/scheduler/batch-login-node/main.tf b/modules/scheduler/batch-login-node/main.tf index ea19feb7f2..24c9834c76 100644 --- a/modules/scheduler/batch-login-node/main.tf +++ b/modules/scheduler/batch-login-node/main.tf @@ -99,7 +99,7 @@ locals { } module "login_startup_script" { - source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=64bc890" + source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=34bb7250" labels = var.labels project_id = var.project_id deployment_name = var.deployment_name From 62b50abc09a02679704e55ae1e03d8ea2c81b94c Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Mon, 27 Mar 2023 15:36:15 -0700 Subject: [PATCH 065/100] Changed qsim example to use T4s on us-central1 --- community/examples/quantum-circuit-simulator.yaml | 9 ++++++--- examples/README.md | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/community/examples/quantum-circuit-simulator.yaml b/community/examples/quantum-circuit-simulator.yaml index 1a351f3e49..ce4c05d51d 100644 --- a/community/examples/quantum-circuit-simulator.yaml +++ b/community/examples/quantum-circuit-simulator.yaml @@ -20,8 +20,8 @@ blueprint_name: quantum-circuit vars: project_id: ## Set project id here deployment_name: qsim-demo - region: us-west4 - zone: us-west4-b + region: us-central1 + zone: us-central1-f # Documentation for each of the modules used below can be found at # https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md @@ -125,7 +125,10 @@ deployment_groups: - network1 - quantum-simulator-setup settings: - machine_type: a2-highgpu-1g + machine_type: n1-standard-32 + guest_accelerator: + - type: nvidia-tesla-t4 + count: 1 instance_image: project: ubuntu-os-cloud family: ubuntu-2004-lts diff --git a/examples/README.md b/examples/README.md index c79f0893b0..b3cb3a0f49 100644 --- a/examples/README.md +++ b/examples/README.md @@ -730,7 +730,7 @@ walks through the use of this blueprint. ### [quantum-circuit-simulator.yaml] ![community-badge] -This blueprint provisions an [A2 series VM with NVIDIA A100 GPU accelerator][a2] +This blueprint provisions a [N1 series VM with NVIDIA T4 GPU accelerator][t4] and compiles [qsim], a [Google Quantum AI][gqai]-developed tool that simulates quantum circuits using CPUs and GPUs. The installation of qsim, the [CUDA Toolkit][cudatk], and the [cuQuantum SDK][cqsdk] is fully automated but takes a @@ -744,7 +744,7 @@ python /var/tmp/qsim-example.py [gqai]: https://quantumai.google/ [quantum-circuit-simulator.yaml]: ../community/examples/quantum-circuit-simulator.yaml -[a2]: https://cloud.google.com/compute/docs/gpus#a100-gpus +[t4]: https://cloud.google.com/compute/docs/gpus#nvidia_t4_gpus [qsim]: https://quantumai.google/qsim [cqsdk]: https://developer.nvidia.com/cuquantum-sdk [cudatk]: https://developer.nvidia.com/cuda-toolkit From 9f2ebfb088b89096a51d35eb8577e6ddde8de40f Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Mon, 27 Mar 2023 21:11:07 -0700 Subject: [PATCH 066/100] chaning CRD test zone to us-central1-f --- tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml | 2 +- tools/validate_configs/test_configs/remote-desktop.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml b/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml index e79ddfe876..4d51692f84 100644 --- a/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml +++ b/tools/cloud-build/daily-tests/tests/chrome-remote-desktop.yml @@ -14,7 +14,7 @@ --- test_name: chrome-remote-desktop deployment_name: chrome-remote-desktop-{{ build }} -zone: us-central1-c +zone: us-central1-f workspace: /workspace blueprint_yaml: "{{ workspace }}/tools/validate_configs/test_configs/remote-desktop.yaml" network: "{{ deployment_name }}-net" diff --git a/tools/validate_configs/test_configs/remote-desktop.yaml b/tools/validate_configs/test_configs/remote-desktop.yaml index f18b6d0902..28b307a9b3 100644 --- a/tools/validate_configs/test_configs/remote-desktop.yaml +++ b/tools/validate_configs/test_configs/remote-desktop.yaml @@ -20,7 +20,7 @@ vars: project_id: ## Set GCP Project ID Here ## deployment_name: remote-desktop region: us-central1 - zone: us-central1-c + zone: us-central1-f deployment_groups: - group: primary From 8da47ef63641fe9e8fc3bfd94f6cf016659daaab Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Mon, 27 Mar 2023 22:17:59 +0000 Subject: [PATCH 067/100] Add flag to skip validators; Check validations during `make tests` * Add `Skip bool` to `validatorConfig`; * Add flag `skip-validators` to `create` and `expand` commangs; * Change validation level IGNORE -> ERROR in validate_configs.sh; * Add default validators even if others are present; ``` $ make && ./ghpc expand examples/hpc-cluster-small.yaml --vars="project_id=GG" --skip-validators="test_project_exists,test_apis_enabled" ``` expanded.yaml: ``` ... validators: - validator: test_project_exists inputs: {} skip: true - validator: test_apis_enabled inputs: {} skip: true - validator: test_module_not_used inputs: {} skip: false ... ``` --- cmd/create.go | 24 +++++++-- cmd/expand.go | 7 ++- docs/blueprint-validation.md | 22 ++++++-- pkg/config/config.go | 20 +++++++ pkg/config/config_test.go | 61 +++++++++++++++++++--- pkg/config/expand.go | 43 ++++++++------- pkg/config/validate.go | 46 +++++++++------- tools/validate_configs/validate_configs.sh | 6 ++- 8 files changed, 174 insertions(+), 55 deletions(-) diff --git a/cmd/create.go b/cmd/create.go index 26ac31071c..4f2cb905ca 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -41,8 +41,8 @@ func init() { "Sets the output directory where the HPC deployment directory will be created.") createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars) createCmd.Flags().StringSliceVar(&cliBEConfigVars, "backend-config", nil, msgCLIBackendConfig) - createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", - validationLevelDesc) + createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", validationLevelDesc) + createCmd.Flags().StringSliceVar(&validatorsToSkip, "skip-validators", nil, skipValidatorsDesc) createCmd.Flags().BoolVarP(&overwriteDeployment, "overwrite-deployment", "w", false, "If specified, an existing deployment directory is overwritten by the new deployment. \n"+ "Note: Terraform state IS preserved. \n"+ @@ -60,7 +60,10 @@ var ( overwriteDeployment bool validationLevel string validationLevelDesc = "Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")" - createCmd = &cobra.Command{ + validatorsToSkip []string + skipValidatorsDesc = "Validators to skip" + + createCmd = &cobra.Command{ Use: "create BLUEPRINT_NAME", Short: "Create a new deployment.", Long: "Create a new deployment based on a provided blueprint.", @@ -92,6 +95,9 @@ func runCreateCmd(cmd *cobra.Command, args []string) { if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil { log.Fatal(err) } + if err := skipValidators(&deploymentConfig); err != nil { + log.Fatal(err) + } if err := deploymentConfig.ExpandConfig(); err != nil { log.Fatal(err) } @@ -105,3 +111,15 @@ func runCreateCmd(cmd *cobra.Command, args []string) { } } } + +func skipValidators(dc *config.DeploymentConfig) error { + if validatorsToSkip == nil { + return nil + } + for _, v := range validatorsToSkip { + if err := dc.SkipValidator(v); err != nil { + return err + } + } + return nil +} diff --git a/cmd/expand.go b/cmd/expand.go index ebfa6c3e85..6a43e0dd3c 100644 --- a/cmd/expand.go +++ b/cmd/expand.go @@ -33,8 +33,8 @@ func init() { "Output file for the expanded HPC Environment Definition.") expandCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars) expandCmd.Flags().StringSliceVar(&cliBEConfigVars, "backend-config", nil, msgCLIBackendConfig) - expandCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", - validationLevelDesc) + expandCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", validationLevelDesc) + expandCmd.Flags().StringSliceVar(&validatorsToSkip, "skip-validators", nil, skipValidatorsDesc) rootCmd.AddCommand(expandCmd) } @@ -72,6 +72,9 @@ func runExpandCmd(cmd *cobra.Command, args []string) { if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil { log.Fatal(err) } + if err := skipValidators(&deploymentConfig); err != nil { + log.Fatal(err) + } if err := deploymentConfig.ExpandConfig(); err != nil { log.Fatal(err) } diff --git a/docs/blueprint-validation.md b/docs/blueprint-validation.md index 732a95b320..d8a9d59378 100644 --- a/docs/blueprint-validation.md +++ b/docs/blueprint-validation.md @@ -81,13 +81,29 @@ Each validator is described below: ### Explicit validators Validators can be overwritten and supplied with alternative input values, -however they are limited to the set of functions defined above. One method by -which to disable validators is to explicitly set them to the empty list: +however they are limited to the set of functions defined above. + +### Skipping or disabling validators + +There are three methods to disable configured validators: + +* Set `skip` value in validator config: ```yaml -validators: [] +validators: +- validator: test_apis_enabled + inputs: {} + skip: true ``` +* Use `skip-validators` CLI flag: + +```shell +./ghpc create ... --skip-validators="test_project_exists,test_apis_enabled" +``` + +* To disable all validators, set the [validation level to IGNORE](#validation-levels). + ### Validation levels They can also be set to 3 differing levels of behavior using the command-line diff --git a/pkg/config/config.go b/pkg/config/config.go index 338a286ffe..d4ac0bd66e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -184,6 +184,7 @@ func (v validatorName) String() string { type validatorConfig struct { Validator string Inputs map[string]interface{} + Skip bool } func (v *validatorConfig) check(name validatorName, requiredInputs []string) error { @@ -655,6 +656,25 @@ func (dc *DeploymentConfig) SetBackendConfig(cliBEConfigVars []string) error { return nil } +// SkipValidator marks validator(s) as skipped, +// if no validator is present, adds one, marked as skipped. +func (dc *DeploymentConfig) SkipValidator(name string) error { + if dc.Config.Validators == nil { + dc.Config.Validators = []validatorConfig{} + } + skipped := false + for i, v := range dc.Config.Validators { + if v.Validator == name { + dc.Config.Validators[i].Skip = true + skipped = true + } + } + if !skipped { + dc.Config.Validators = append(dc.Config.Validators, validatorConfig{Validator: name, Skip: true}) + } + return nil +} + // IsLiteralVariable returns true if string matches variable ((ctx.name)) func IsLiteralVariable(str string) bool { return literalExp.MatchString(str) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 095d94a60b..87ce7f280d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -955,19 +955,13 @@ func (s *MySuite) TestValidatorConfigCheck(c *C) { const vn = testProjectExistsName // some valid name { // FAIL: names mismatch - v := validatorConfig{ - "who_is_this", - map[string]interface{}{}, - } + v := validatorConfig{"who_is_this", map[string]interface{}{}, false} err := v.check(vn, []string{}) c.Check(err, ErrorMatches, "passed wrong validator to test_project_exists implementation") } { // OK: names match - v := validatorConfig{ - vn.String(), - map[string]interface{}{}, - } + v := validatorConfig{vn.String(), map[string]interface{}{}, false} c.Check(v.check(vn, []string{}), IsNil) } @@ -975,6 +969,7 @@ func (s *MySuite) TestValidatorConfigCheck(c *C) { v := validatorConfig{ vn.String(), map[string]interface{}{"in0": nil, "in1": nil}, + false, } c.Check(v.check(vn, []string{"in0", "in1"}), IsNil) c.Check(v.check(vn, []string{"in1", "in0"}), IsNil) @@ -984,6 +979,7 @@ func (s *MySuite) TestValidatorConfigCheck(c *C) { v := validatorConfig{ vn.String(), map[string]interface{}{"in0": nil, "in1": nil}, + false, } err := v.check(vn, []string{"in0", "in1", "in2"}) c.Check(err, ErrorMatches, missingRequiredInputRegex) @@ -993,6 +989,7 @@ func (s *MySuite) TestValidatorConfigCheck(c *C) { v := validatorConfig{ vn.String(), map[string]interface{}{"in0": nil, "in1": nil, "in3": nil}, + false, } err := v.check(vn, []string{"in0", "in1", "in2"}) c.Check(err, ErrorMatches, missingRequiredInputRegex) @@ -1002,6 +999,7 @@ func (s *MySuite) TestValidatorConfigCheck(c *C) { v := validatorConfig{ vn.String(), map[string]interface{}{"in0": nil, "in1": nil, "in2": nil, "in3": nil}, + false, } err := v.check(vn, []string{"in0", "in1", "in2"}) c.Check(err, ErrorMatches, "only 3 inputs \\[in0 in1 in2\\] should be provided to test_project_exists") @@ -1087,3 +1085,50 @@ func (s *MySuite) TestCheckBackends(c *C) { c.Check(check(b), IsNil) } } + +func (s *MySuite) TestSkipValidator(c *C) { + { + dc := DeploymentConfig{Config: Blueprint{Validators: nil}} + c.Check(dc.SkipValidator("zebra"), IsNil) + c.Check(dc.Config.Validators, DeepEquals, []validatorConfig{ + {Validator: "zebra", Skip: true}}) + } + { + dc := DeploymentConfig{Config: Blueprint{Validators: []validatorConfig{ + {Validator: "pony"}}}} + c.Check(dc.SkipValidator("zebra"), IsNil) + c.Check(dc.Config.Validators, DeepEquals, []validatorConfig{ + {Validator: "pony"}, + {Validator: "zebra", Skip: true}}) + } + { + dc := DeploymentConfig{Config: Blueprint{Validators: []validatorConfig{ + {Validator: "pony"}, + {Validator: "zebra"}}}} + c.Check(dc.SkipValidator("zebra"), IsNil) + c.Check(dc.Config.Validators, DeepEquals, []validatorConfig{ + {Validator: "pony"}, + {Validator: "zebra", Skip: true}}) + } + { + dc := DeploymentConfig{Config: Blueprint{Validators: []validatorConfig{ + {Validator: "pony"}, + {Validator: "zebra", Skip: true}}}} + c.Check(dc.SkipValidator("zebra"), IsNil) + c.Check(dc.Config.Validators, DeepEquals, []validatorConfig{ + {Validator: "pony"}, + {Validator: "zebra", Skip: true}}) + } + { + dc := DeploymentConfig{Config: Blueprint{Validators: []validatorConfig{ + {Validator: "zebra"}, + {Validator: "pony"}, + {Validator: "zebra"}}}} + c.Check(dc.SkipValidator("zebra"), IsNil) + c.Check(dc.Config.Validators, DeepEquals, []validatorConfig{ + {Validator: "zebra", Skip: true}, + {Validator: "pony"}, + {Validator: "zebra", Skip: true}}) + } + +} diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 2e64858329..58e4bf4640 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -980,19 +980,19 @@ func (dc *DeploymentConfig) expandVariables() error { return nil } -// this function adds default validators to the blueprint if none have been -// defined. default validators are only added for global variables that exist +// this function adds default validators to the blueprint. +// default validators are only added for global variables that exist func (dc *DeploymentConfig) addDefaultValidators() error { - if dc.Config.Validators != nil { - return nil + if dc.Config.Validators == nil { + dc.Config.Validators = []validatorConfig{} } - dc.Config.Validators = []validatorConfig{} _, projectIDExists := dc.Config.Vars["project_id"] _, regionExists := dc.Config.Vars["region"] _, zoneExists := dc.Config.Vars["zone"] - dc.Config.Validators = append(dc.Config.Validators, validatorConfig{ + defaults := []validatorConfig{} + defaults = append(defaults, validatorConfig{ Validator: testModuleNotUsedName.String(), Inputs: map[string]interface{}{}, }) @@ -1001,53 +1001,60 @@ func (dc *DeploymentConfig) addDefaultValidators() error { // only succeed if credentials can access the project. If the project ID // validator fails, all remaining validators are not executed. if projectIDExists { - v := validatorConfig{ + defaults = append(defaults, validatorConfig{ Validator: testProjectExistsName.String(), Inputs: map[string]interface{}{ "project_id": "$(vars.project_id)", }, - } - dc.Config.Validators = append(dc.Config.Validators, v) + }) } // it is safe to run this validator even if vars.project_id is undefined; // it will likely fail but will do so helpfully to the user - dc.Config.Validators = append(dc.Config.Validators, validatorConfig{ + defaults = append(defaults, validatorConfig{ Validator: "test_apis_enabled", Inputs: map[string]interface{}{}, }) if projectIDExists && regionExists { - v := validatorConfig{ + defaults = append(defaults, validatorConfig{ Validator: testRegionExistsName.String(), Inputs: map[string]interface{}{ "project_id": "$(vars.project_id)", "region": "$(vars.region)", }, - } - dc.Config.Validators = append(dc.Config.Validators, v) - + }) } if projectIDExists && zoneExists { - v := validatorConfig{ + defaults = append(defaults, validatorConfig{ Validator: testZoneExistsName.String(), Inputs: map[string]interface{}{ "project_id": "$(vars.project_id)", "zone": "$(vars.zone)", }, - } - dc.Config.Validators = append(dc.Config.Validators, v) + }) } if projectIDExists && regionExists && zoneExists { - v := validatorConfig{ + defaults = append(defaults, validatorConfig{ Validator: testZoneInRegionName.String(), Inputs: map[string]interface{}{ "project_id": "$(vars.project_id)", "region": "$(vars.region)", "zone": "$(vars.zone)", }, + }) + } + + used := map[string]bool{} + for _, v := range dc.Config.Validators { + used[v.Validator] = true + } + + for _, v := range defaults { + if used[v.Validator] { + continue } dc.Config.Validators = append(dc.Config.Validators, v) } diff --git a/pkg/config/validate.go b/pkg/config/validate.go index fead056ade..e09e4b7457 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -82,30 +82,36 @@ func (dc DeploymentConfig) executeValidators() error { } for _, validator := range dc.Config.Validators { - if f, ok := implementedValidators[validator.Validator]; ok { - err := f(validator) - if err != nil { - var prefix string - switch dc.Config.ValidationLevel { - case validationWarning: - warned = true - prefix = "warning: " - default: - errored = true - prefix = "error: " - } - log.Print(prefix, err) - log.Println() + if validator.Skip { + continue + } - // do not bother running further validators if project ID could not be found - if validator.Validator == testProjectExistsName.String() { - break - } - } - } else { + f, ok := implementedValidators[validator.Validator] + if !ok { errored = true log.Printf("%s is not an implemented validator", validator.Validator) + continue } + + if err := f(validator); err != nil { + var prefix string + switch dc.Config.ValidationLevel { + case validationWarning: + warned = true + prefix = "warning: " + default: + errored = true + prefix = "error: " + } + log.Print(prefix, err) + log.Println() + + // do not bother running further validators if project ID could not be found + if validator.Validator == testProjectExistsName.String() { + break + } + } + } if warned || errored { diff --git a/tools/validate_configs/validate_configs.sh b/tools/validate_configs/validate_configs.sh index 2ea3e74c8c..46e24be1b9 100755 --- a/tools/validate_configs/validate_configs.sh +++ b/tools/validate_configs/validate_configs.sh @@ -21,6 +21,7 @@ run_test() { exampleFile=$(basename "$example") DEPLOYMENT=$(echo "${exampleFile%.yaml}-$(basename "${tmpdir##*.}")" | sed -e 's/\(.*\)/\L\1/') PROJECT="invalid-project" + VALIDATORS_TO_SKIP="test_project_exists,test_apis_enabled,test_region_exists,test_zone_exists,test_zone_in_region" GHPC_PATH="${cwd}/ghpc" # Cover the three possible starting sequences for local sources: ./ ../ / LOCAL_SOURCE_PATTERN='source:\s\+\(\./\|\.\./\|/\)' @@ -35,7 +36,10 @@ run_test() { else cd "${tmpdir}" fi - ${GHPC_PATH} create -l IGNORE --vars "project_id=${PROJECT},deployment_name=${DEPLOYMENT}" "${tmpdir}"/"${exampleFile}" >/dev/null || + ${GHPC_PATH} create -l ERROR \ + --skip-validators="${VALIDATORS_TO_SKIP}" \ + --vars="project_id=${PROJECT},deployment_name=${DEPLOYMENT}" \ + "${tmpdir}"/"${exampleFile}" >/dev/null || { echo "*** ERROR: error creating deployment with ghpc for ${exampleFile}" exit 1 From c5b8bfc9357e5029eae4ee42ea0e98e8df4d360b Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Mon, 27 Mar 2023 22:19:11 +0000 Subject: [PATCH 068/100] Fix typo test_region_exists -> test_zone_exists --- docs/blueprint-validation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/blueprint-validation.md b/docs/blueprint-validation.md index d8a9d59378..9faa3463e7 100644 --- a/docs/blueprint-validation.md +++ b/docs/blueprint-validation.md @@ -64,7 +64,7 @@ Each validator is described below: * FAIL: if region does not exist or is not accessible within the project * Typical failures involve simple typos * Manual test: `gcloud compute regions describe $(vars.region) --project $(vars.project_id)` -* `test_region_exists` +* `test_zone_exists` * Inputs: `zone` (string) * PASS: if zone exists and is accessible within the project * FAIL: if zone does not exist or is not accessible within the project From ec5b77d6c6f9e025166b5099009445322a0d06da Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Tue, 28 Mar 2023 00:31:54 +0000 Subject: [PATCH 069/100] Unify network_storage variables type Use ``` list(object({ server_ip = string remote_mount = string local_mount = string fs_type = string mount_options = string client_install_runner = map(string) mount_runner = map(string) })) ``` --- .../SchedMD-slurm-on-gcp-partition/README.md | 2 +- .../variables.tf | 12 ++++--- .../compute/htcondor-execute-point/README.md | 2 +- .../htcondor-execute-point/variables.tf | 12 ++++--- .../schedmd-slurm-gcp-v5-partition/README.md | 2 +- .../variables.tf | 12 ++++--- .../SchedMD-slurm-on-gcp-controller/README.md | 2 +- .../variables.tf | 12 ++++--- .../SchedMD-slurm-on-gcp-login-node/README.md | 2 +- .../variables.tf | 13 +++++--- .../schedmd-slurm-gcp-v5-controller/README.md | 2 +- .../variables.tf | 21 +++++------- .../schedmd-slurm-gcp-v5-hybrid/README.md | 2 +- .../schedmd-slurm-gcp-v5-hybrid/variables.tf | 21 +++++------- pkg/inspect/modules_test.go | 33 ++++++++++++++++--- pkg/modulereader/hcl_utils.go | 8 ++--- pkg/modulereader/hcl_utils_test.go | 12 +++---- 17 files changed, 98 insertions(+), 72 deletions(-) diff --git a/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md b/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md index d1c6d5db1c..6b7b2a865e 100644 --- a/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md +++ b/community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md @@ -77,7 +77,7 @@ No resources. | [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no | | [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes | `string` | `"c2-standard-60"` | no | | [max\_node\_count](#input\_max\_node\_count) | Maximum number of nodes allowed in this partition | `number` | `50` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [partition\_name](#input\_partition\_name) | The name of the slurm partition | `string` | n/a | yes | | [preemptible\_bursting](#input\_preemptible\_bursting) | Should use preemptibles to burst | `string` | `false` | no | | [regional\_capacity](#input\_regional\_capacity) | If True, then create instances in the region that has available capacity. Specify the region in the zone field. | `bool` | `false` | no | diff --git a/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf b/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf index ef21cda036..adca7bf127 100644 --- a/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf +++ b/community/modules/compute/SchedMD-slurm-on-gcp-partition/variables.tf @@ -112,11 +112,13 @@ variable "gpu_type" { variable "network_storage" { description = "An array of network attached storage mounts to be configured on the partition compute nodes." type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) })) default = [] } diff --git a/community/modules/compute/htcondor-execute-point/README.md b/community/modules/compute/htcondor-execute-point/README.md index 8ccf4f61a3..b660a3247d 100644 --- a/community/modules/compute/htcondor-execute-point/README.md +++ b/community/modules/compute/htcondor-execute-point/README.md @@ -188,7 +188,7 @@ No resources. | [metadata](#input\_metadata) | Metadata to add to HTCondor execute points | `map(string)` | `{}` | no | | [min\_idle](#input\_min\_idle) | Minimum number of idle VMs in the HTCondor pool (if pool reaches var.max\_size, this minimum is not guaranteed); set to ensure jobs beginning run more quickly. | `number` | `0` | no | | [network\_self\_link](#input\_network\_self\_link) | The self link of the network HTCondor execute points will join | `string` | `"default"` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [project\_id](#input\_project\_id) | Project in which the HTCondor execute points will be created | `string` | n/a | yes | | [region](#input\_region) | The region in which HTCondor execute points will be created | `string` | n/a | yes | | [service\_account](#input\_service\_account) | Service account to attach to HTCondor execute points | 
object({
    email  = string,
    scopes = set(string)
  })
| 
{
  "email": null,
  "scopes": [
    "https://www.googleapis.com/auth/cloud-platform"
  ]
}
| no | diff --git a/community/modules/compute/htcondor-execute-point/variables.tf b/community/modules/compute/htcondor-execute-point/variables.tf index 37a7f476bf..93037c5a86 100644 --- a/community/modules/compute/htcondor-execute-point/variables.tf +++ b/community/modules/compute/htcondor-execute-point/variables.tf @@ -54,11 +54,13 @@ variable "startup_script" { variable "network_storage" { description = "An array of network attached storage mounts to be configured" type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) })) default = [] } diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md b/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md index 847556acfd..3d3ba38ad3 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/README.md @@ -174,7 +174,7 @@ limitations under the License. | [enable\_reconfigure](#input\_enable\_reconfigure) | Enables automatic Slurm reconfigure on when Slurm configuration changes (e.g.
slurm.conf.tpl, partition details). Compute instances and resource policies
(e.g. placement groups) will be destroyed to align with new configuration.

NOTE: Requires Python and Google Pub/Sub API.

*WARNING*: Toggling this will impact the running workload. Deployed compute nodes
will be destroyed and their jobs will be requeued. | `bool` | `false` | no | | [exclusive](#input\_exclusive) | Exclusive job access to nodes. | `bool` | `true` | no | | [is\_default](#input\_is\_default) | Sets this partition as the default partition by updating the partition\_conf.
If "Default" is already set in partition\_conf, this variable will have no effect. | `bool` | `false` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [node\_groups](#input\_node\_groups) | A list of node groups associated with this partition. See
schedmd-slurm-gcp-v5-node-group for more information on defining a node
group in a blueprint. | 
list(object({
    access_config = list(object({
      network_tier = string
    }))
    node_count_static      = number
    node_count_dynamic_max = number
    group_name             = string
    node_conf              = map(string)
    additional_disks = list(object({
      disk_name    = string
      device_name  = string
      disk_size_gb = number
      disk_type    = string
      disk_labels  = map(string)
      auto_delete  = bool
      boot         = bool
    }))
    bandwidth_tier         = string
    can_ip_forward         = bool
    disable_smt            = bool
    disk_auto_delete       = bool
    disk_labels            = map(string)
    disk_size_gb           = number
    disk_type              = string
    enable_confidential_vm = bool
    enable_oslogin         = bool
    enable_shielded_vm     = bool
    enable_spot_vm         = bool
    gpu = object({
      count = number
      type  = string
    })
    instance_template   = string
    labels              = map(string)
    machine_type        = string
    metadata            = map(string)
    min_cpu_platform    = string
    on_host_maintenance = string
    preemptible         = bool
    service_account = object({
      email  = string
      scopes = list(string)
    })
    shielded_instance_config = object({
      enable_integrity_monitoring = bool
      enable_secure_boot          = bool
      enable_vtpm                 = bool
    })
    spot_instance_config = object({
      termination_action = string
    })
    source_image_family  = string
    source_image_project = string
    source_image         = string
    tags                 = list(string)
  }))
| `[]` | no | | [partition\_conf](#input\_partition\_conf) | Slurm partition configuration as a map.
See https://slurm.schedmd.com/slurm.conf.html#SECTION_PARTITION-CONFIGURATION | `map(string)` | `{}` | no | | [partition\_name](#input\_partition\_name) | The name of the slurm partition. | `string` | n/a | yes | diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf index 89ef0de009..079a175b96 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/variables.tf @@ -189,11 +189,13 @@ variable "enable_reconfigure" { variable "network_storage" { description = "An array of network attached storage mounts to be configured on the partition compute nodes." type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) })) default = [] } diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md index d4abf9ce7f..6befbf60a0 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/README.md @@ -112,7 +112,7 @@ No resources. | [labels](#input\_labels) | Labels to add to controller instance. Key-value pairs. | `map(string)` | `{}` | no | | [login\_node\_count](#input\_login\_node\_count) | Number of login nodes in the cluster | `number` | `0` | no | | [munge\_key](#input\_munge\_key) | Specific munge key to use | `any` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip     = string,
    remote_mount  = string,
    local_mount   = string,
    fs_type       = string,
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [partition](#input\_partition) | An array of configurations for specifying multiple machine types residing in their own Slurm partitions. | 
list(object({
    name                 = string,
    machine_type         = string,
    max_node_count       = number,
    zone                 = string,
    image                = string,
    image_hyperthreads   = bool,
    compute_disk_type    = string,
    compute_disk_size_gb = number,
    compute_labels       = any,
    cpu_platform         = string,
    gpu_type             = string,
    gpu_count            = number,
    network_storage = list(object({
      server_ip     = string,
      remote_mount  = string,
      local_mount   = string,
      fs_type       = string,
      mount_options = string
    })),
    preemptible_bursting = string,
    vpc_subnet           = string,
    exclusive            = bool,
    enable_placement     = bool,
    regional_capacity    = bool,
    regional_policy      = any,
    instance_template    = string,
    bandwidth_tier       = string,
    static_node_count    = number
  }))
| n/a | yes | | [project\_id](#input\_project\_id) | Compute Platform project that will host the Slurm cluster | `string` | n/a | yes | | [region](#input\_region) | Compute Platform region where the Slurm cluster will be located | `string` | n/a | yes | diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf index 492a4e09a2..315e0352fb 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/variables.tf @@ -142,11 +142,13 @@ variable "jwt_key" { variable "network_storage" { description = "An array of network attached storage mounts to be configured on all instances." type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) })) default = [] } diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md index aca6750e3b..e73c7a408b 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/README.md @@ -105,7 +105,7 @@ No resources. | [login\_service\_account](#input\_login\_service\_account) | Service Account for compute nodes. | `string` | `null` | no | | [login\_startup\_script](#input\_login\_startup\_script) | Custom startup script to run on the login node | `string` | `null` | no | | [munge\_key](#input\_munge\_key) | Specific munge key to use | `any` | `null` | no | -| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip    = string,
    remote_mount = string,
    local_mount  = string,
    fs_type      = string,
  mount_options = string }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [region](#input\_region) | Compute Platform region where the Slurm cluster will be located | `string` | n/a | yes | | [shared\_vpc\_host\_project](#input\_shared\_vpc\_host\_project) | Host project of shared VPC | `string` | `null` | no | | [startup\_script](#input\_startup\_script) | Custom startup script to run on the login node.
Will be ignored if `login_startup_script` is specified.
This variable allows Slurm to [use](https://github.com/GoogleCloudPlatform/hpc-toolkit/tree/main/modules#use-optional) the [startup\_script](https://github.com/GoogleCloudPlatform/hpc-toolkit/tree/main/modules/scripts/startup-script) module. | `string` | `null` | no | diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf index d4e455a2ca..79768d6bde 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/variables.tf @@ -107,11 +107,14 @@ variable "munge_key" { variable "network_storage" { description = " An array of network attached storage mounts to be configured on all instances." type = list(object({ - server_ip = string, - remote_mount = string, - local_mount = string, - fs_type = string, - mount_options = string })) + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) + })) default = [] } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md index a0970c9866..12558d4e6d 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/README.md @@ -214,7 +214,7 @@ limitations under the License. | [min\_cpu\_platform](#input\_min\_cpu\_platform) | Specifies a minimum CPU platform. Applicable values are the friendly names of
CPU platforms, such as Intel Haswell or Intel Skylake. See the complete list:
https://cloud.google.com/compute/docs/instances/specify-min-cpu-platform | `string` | `null` | no | | [network\_ip](#input\_network\_ip) | Private IP address to assign to the instance if desired. | `string` | `""` | no | | [network\_self\_link](#input\_network\_self\_link) | Network to deploy to. Either network\_self\_link or subnetwork\_self\_link must be specified. | `string` | `null` | no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
server\_ip : Address of the storage server.
remote\_mount : The location in the remote instance filesystem to mount from.
local\_mount : The location on the instance filesystem to mount to.
fs\_type : Filesystem type (e.g. "nfs").
mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [on\_host\_maintenance](#input\_on\_host\_maintenance) | Instance availability Policy. | `string` | `"MIGRATE"` | no | | [partition](#input\_partition) | Cluster partitions as a list. | 
list(object({
    compute_list = list(string)
    partition = object({
      enable_job_exclusive    = bool
      enable_placement_groups = bool
      network_storage = list(object({
        server_ip     = string
        remote_mount  = string
        local_mount   = string
        fs_type       = string
        mount_options = string
      }))
      partition_conf = map(string)
      partition_name = string
      partition_nodes = map(object({
        access_config = list(object({
          network_tier = string
        }))
        bandwidth_tier         = string
        node_count_dynamic_max = number
        node_count_static      = number
        enable_spot_vm         = bool
        group_name             = string
        instance_template      = string
        node_conf              = map(string)
        spot_instance_config = object({
          termination_action = string
        })
      }))
      partition_startup_scripts_timeout = number
      subnetwork                        = string
      zone_policy_allow                 = list(string)
      zone_policy_deny                  = list(string)
      zone_target_shape                 = string
    })
  }))
| `[]` | no | | [preemptible](#input\_preemptible) | Allow the instance to be preempted. | `bool` | `false` | no | diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf index 6651b68e4b..62b1b0fadb 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/variables.tf @@ -350,20 +350,15 @@ variable "network_ip" { } variable "network_storage" { - description = < [google\_app\_cred\_path](#input\_google\_app\_cred\_path) | Path to Google Applicaiton Credentials. | `string` | `null` | no | | [install\_dir](#input\_install\_dir) | Directory where the hybrid configuration directory will be installed on the
on-premise controller. This updates the prefix path for the resume and
suspend scripts in the generated `cloud.conf` file. The value defaults to
output\_dir if not specified. | `string` | `null` | no | | [munge\_mount](#input\_munge\_mount) | Remote munge mount for compute and login nodes to acquire the munge.key.

By default, the munge mount server will be assumed to be the
`var.slurm_control_host` (or `var.slurm_control_addr` if non-null) when
`server_ip=null`. | 
object({
    server_ip     = string
    remote_mount  = string
    fs_type       = string
    mount_options = string
  })
| 
{
  "fs_type": "nfs",
  "mount_options": "",
  "remote_mount": "/etc/munge/",
  "server_ip": null
}
| no | -| [network\_storage](#input\_network\_storage) | Storage to mounted on all instances.
- server\_ip : Address of the storage server.
- remote\_mount : The location in the remote instance filesystem to mount from.
- local\_mount : The location on the instance filesystem to mount to.
- fs\_type : Filesystem type (e.g. "nfs").
- mount\_options : Options to mount with. | 
list(object({
    server_ip     = string
    remote_mount  = string
    local_mount   = string
    fs_type       = string
    mount_options = string
  }))
| `[]` | no | +| [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on all instances. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [output\_dir](#input\_output\_dir) | Directory where this module will write its files to. These files include:
cloud.conf; cloud\_gres.conf; config.yaml; resume.py; suspend.py; and util.py.
If not specified explicitly, this will also be used as the default value
for the `install_dir` variable. | `string` | `null` | no | | [partition](#input\_partition) | Cluster partitions as a list. | 
list(object({
    compute_list = list(string)
    partition = object({
      enable_job_exclusive    = bool
      enable_placement_groups = bool
      network_storage = list(object({
        server_ip     = string
        remote_mount  = string
        local_mount   = string
        fs_type       = string
        mount_options = string
      }))
      partition_conf = map(string)
      partition_name = string
      partition_nodes = map(object({
        bandwidth_tier         = string
        node_count_dynamic_max = number
        node_count_static      = number
        enable_spot_vm         = bool
        group_name             = string
        instance_template      = string
        node_conf              = map(string)
        access_config = list(object({
          network_tier = string
        }))
        spot_instance_config = object({
          termination_action = string
        })
      }))
      partition_startup_scripts_timeout = number
      subnetwork                        = string
      zone_policy_allow                 = list(string)
      zone_policy_deny                  = list(string)
      zone_target_shape                 = string
    })
  }))
| `[]` | no | | [project\_id](#input\_project\_id) | Project ID to create resources in. | `string` | n/a | yes | diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/variables.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/variables.tf index d0593e6a80..34e509d2e7 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/variables.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-hybrid/variables.tf @@ -189,20 +189,15 @@ variable "disable_default_mounts" { } variable "network_storage" { - description = <<-EOD - Storage to mounted on all instances. - - server_ip : Address of the storage server. - - remote_mount : The location in the remote instance filesystem to mount from. - - local_mount : The location on the instance filesystem to mount to. - - fs_type : Filesystem type (e.g. "nfs"). - - mount_options : Options to mount with. - EOD + description = "An array of network attached storage mounts to be configured on all instances." type = list(object({ - server_ip = string - remote_mount = string - local_mount = string - fs_type = string - mount_options = string + server_ip = string, + remote_mount = string, + local_mount = string, + fs_type = string, + mount_options = string, + client_install_runner = map(string) + mount_runner = map(string) })) default = [] } diff --git a/pkg/inspect/modules_test.go b/pkg/inspect/modules_test.go index 38acf03a9c..f2f4e7efb2 100644 --- a/pkg/inspect/modules_test.go +++ b/pkg/inspect/modules_test.go @@ -102,11 +102,36 @@ func TestSanity(t *testing.T) { notEmpty(query(all()), t) } +func checkInputType(t *testing.T, mod modInfo, input string, expected string) { + i, ok := mod.Input(input) + if !ok { + t.Errorf("%s does not have input %s", mod.Source, input) + } + expected = modulereader.NormalizeType(expected) + got := modulereader.NormalizeType(i.Type) + if expected != got { + t.Errorf("%s %s has unexpected type expected:\n%#v\ngot:\n%#v", + mod.Source, input, expected, got) + } +} + func TestLabelsType(t *testing.T) { for _, mod := range notEmpty(query(hasInput("labels")), t) { - labels, _ := mod.Input("labels") - if labels.Type != "map(string)" { - t.Errorf("%s.labels has unexpected type %#v", mod.Source, labels.Type) - } + checkInputType(t, mod, "labels", "map(string)") + } +} + +func TestNetworkStorage(t *testing.T) { + expected := `list(object({ + server_ip = string + remote_mount = string + local_mount = string + fs_type = string + mount_options = string + client_install_runner = map(string) + mount_runner = map(string) + }))` + for _, mod := range notEmpty(query(hasInput("network_storage")), t) { + checkInputType(t, mod, "network_storage", expected) } } diff --git a/pkg/modulereader/hcl_utils.go b/pkg/modulereader/hcl_utils.go index 9875de4386..fb4a2d2236 100644 --- a/pkg/modulereader/hcl_utils.go +++ b/pkg/modulereader/hcl_utils.go @@ -93,15 +93,15 @@ func getCtyType(hclType string) (cty.Type, error) { return typ, nil } -// Attempts to bring semantically equal types to equal string representation. E.g.: +// NormalizeType attempts to bring semantically equal types to equal string representation. E.g.: // -// normalizeType("object({count=number,kind=string})") -// == normalizeType("object({kind=string,count=number})"). +// NormalizeType("object({count=number,kind=string})") +// == NormalizeType("object({kind=string,count=number})"). // // Additionally it removes any comments, whitespaces and line breaks. // // This method is fail-safe, if error arises passed type will be returned without changes. -func normalizeType(hclType string) string { +func NormalizeType(hclType string) string { ctyType, err := getCtyType(hclType) if err != nil { log.Printf("Failed to parse HCL type='%s', got %v", hclType, err) diff --git a/pkg/modulereader/hcl_utils_test.go b/pkg/modulereader/hcl_utils_test.go index 7763049507..ca140ae03a 100644 --- a/pkg/modulereader/hcl_utils_test.go +++ b/pkg/modulereader/hcl_utils_test.go @@ -20,16 +20,16 @@ import ( func (s *MySuite) TestNormalizeType(c *C) { c.Check( - normalizeType("object({count=number,kind=string})"), + NormalizeType("object({count=number,kind=string})"), Equals, - normalizeType("object({kind=string,count=number})")) + NormalizeType("object({kind=string,count=number})")) - c.Check(normalizeType("?invalid_type"), Equals, "?invalid_type") + c.Check(NormalizeType("?invalid_type"), Equals, "?invalid_type") // `any` is special type, check that it works - c.Check(normalizeType("object({b=any,a=number})"), Equals, normalizeType("object({a=number,b=any})")) + c.Check(NormalizeType("object({b=any,a=number})"), Equals, NormalizeType("object({a=number,b=any})")) - c.Check(normalizeType(" object ( {\na=any\n} ) "), Equals, normalizeType("object({a=any})")) + c.Check(NormalizeType(" object ( {\na=any\n} ) "), Equals, NormalizeType("object({a=any})")) - c.Check(normalizeType(" string # comment"), Equals, normalizeType("string")) + c.Check(NormalizeType(" string # comment"), Equals, NormalizeType("string")) } From 930ae14a69525f1b8c898044bac35e2cd4456525 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 28 Mar 2023 14:37:24 -0500 Subject: [PATCH 070/100] Fix varReference implementation of reference interface --- pkg/config/expand.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 2e64858329..540b3ff2c2 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -711,7 +711,7 @@ func (ref modReference) validate(bp Blueprint) error { // ref.explicitInterGroup: intergroup references must explicitly identify the // target group ID and intragroup references cannot have an incorrect explicit // group ID -func (ref *varReference) validate(bp Blueprint) error { +func (ref varReference) validate(bp Blueprint) error { // simplest case to evaluate is a deployment variable's existence if ref.toGroupID == "deployment" { if ref.toModuleID == "vars" { From aed15b36f85cc2ef798f981cff75a8a8996e9992 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 28 Mar 2023 14:37:24 -0500 Subject: [PATCH 071/100] One-line isUnused func --- pkg/config/config.go | 12 +++--------- pkg/config/config_test.go | 14 ++++---------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 338a286ffe..7a20fbf5be 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -285,14 +285,8 @@ type ModConnection struct { // Returns true if a connection does not functionally link the outputs and // inputs of the modules. This can happen when a module is connected with "use" // but none of the outputs of fromID match the inputs of toID. -func (mc *ModConnection) isEmpty() (isEmpty bool) { - isEmpty = false - if mc.kind == useConnection { - if len(mc.sharedVariables) == 0 { - isEmpty = true - } - } - return +func (mc *ModConnection) isUnused() bool { + return mc.kind == useConnection && len(mc.sharedVariables) == 0 } // DeploymentConfig is a container for the imported YAML data and supporting data for @@ -345,7 +339,7 @@ func (dc *DeploymentConfig) listUnusedModules() map[string][]string { unusedModules := make(map[string][]string) for _, connections := range dc.moduleConnections { for _, conn := range connections { - if conn.isEmpty() { + if conn.isUnused() { fromMod := conn.ref.FromModuleID() unusedModules[fromMod] = append(unusedModules[fromMod], conn.ref.ToModuleID()) } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 095d94a60b..1a10ff42b2 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -368,30 +368,24 @@ func (s *MySuite) TestCheckModuleAndGroupNames(c *C) { } } -func (s *MySuite) TestIsEmpty(c *C) { +func (s *MySuite) TestIsUnused(c *C) { // Use connection is not empty conn := ModConnection{ kind: useConnection, sharedVariables: []string{"var1"}, } - got := conn.isEmpty() - exp := false - c.Assert(got, Equals, exp) + c.Assert(conn.isUnused(), Equals, false) // Use connection is empty conn = ModConnection{ kind: useConnection, sharedVariables: []string{}, } - got = conn.isEmpty() - exp = true - c.Assert(got, Equals, exp) + c.Assert(conn.isUnused(), Equals, true) // Undefined connection kind conn = ModConnection{} - got = conn.isEmpty() - exp = false - c.Assert(got, Equals, exp) + c.Assert(conn.isUnused(), Equals, false) } func (s *MySuite) TestListUnusedModules(c *C) { From a0a993630a65ad222157e79f4f2cc00ad888cb53 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Tue, 28 Mar 2023 14:37:24 -0500 Subject: [PATCH 072/100] Fix varReference and modReference tracking of calling module ID --- pkg/config/config.go | 2 +- pkg/config/expand.go | 10 +++++--- pkg/config/expand_test.go | 54 ++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 25 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 7a20fbf5be..ede916f590 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -532,7 +532,7 @@ func checkUsedModuleNames(bp Blueprint) error { for _, grp := range bp.DeploymentGroups { for _, mod := range grp.Modules { for _, usedMod := range mod.Use { - ref, err := identifyModuleByReference(usedMod, grp) + ref, err := identifyModuleByReference(usedMod, grp, mod) if err != nil { return err } diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 540b3ff2c2..765b39f4b7 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -262,7 +262,7 @@ func (dc *DeploymentConfig) applyUseModules() error { // this will enable us to get structs about the module being // used and search it for outputs that match inputs in the // current module (the iterator) - modRef, err := identifyModuleByReference(toModID, *group) + modRef, err := identifyModuleByReference(toModID, *group, *fromMod) if err != nil { return err } @@ -556,7 +556,7 @@ string into a modReference struct as defined above. An input string consists of 1 or 2 fields or if either field is the empty string. This function does not ensure the existence of the module! */ -func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modReference, error) { +func identifyModuleByReference(yamlReference string, dg DeploymentGroup, fromMod Module) (modReference, error) { // struct defaults: empty strings and false booleans var ref modReference // intra-group references length 1 and inter-group references length 2 @@ -572,6 +572,7 @@ func identifyModuleByReference(yamlReference string, dg DeploymentGroup) (modRef ref.fromGroupID = dg.Name ref.explicit = true } + ref.fromModuleID = fromMod.ID // should consider more sophisticated definition of valid values here. // for now check that no fields are the empty string; due to the default @@ -627,12 +628,13 @@ string into a varReference struct as defined above. An input string consists of 2 or 3 fields, or if any field is the empty string. This function does not ensure the existence of the reference! */ -func identifySimpleVariable(yamlReference string, dg DeploymentGroup) (varReference, error) { +func identifySimpleVariable(yamlReference string, dg DeploymentGroup, fromMod Module) (varReference, error) { varComponents := strings.Split(yamlReference, ".") // struct defaults: empty strings and false booleans var ref varReference ref.fromGroupID = dg.Name + ref.fromModuleID = fromMod.ID // intra-group references length 2 and inter-group references length 3 switch len(varComponents) { @@ -798,7 +800,7 @@ func expandSimpleVariable(context varContext) (string, error) { callingGroup := context.blueprint.DeploymentGroups[context.groupIndex] refStr := contents[1] - varRef, err := identifySimpleVariable(refStr, callingGroup) + varRef, err := identifySimpleVariable(refStr, callingGroup, callingGroup.Modules[context.modIndex]) if err != nil { return "", err } diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index b7f4f7be1e..865647bd93 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -551,34 +551,41 @@ func (s *MySuite) TestIdentifyModuleByReference(c *C) { Name: "zero", } - ref, err = identifyModuleByReference("module_id", dg) + fromMod := Module{ + ID: "from_module_id", + } + + ref, err = identifyModuleByReference("to_module_id", dg, fromMod) c.Assert(err, IsNil) c.Assert(ref.toGroupID, Equals, dg.Name) c.Assert(ref.fromGroupID, Equals, dg.Name) - c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.fromModuleID, Equals, fromMod.ID) + c.Assert(ref.toModuleID, Equals, "to_module_id") c.Assert(ref.explicit, Equals, false) - ref, err = identifyModuleByReference("explicit_group_id.module_id", dg) + ref, err = identifyModuleByReference("explicit_group_id.module_id", dg, fromMod) c.Assert(err, IsNil) c.Assert(ref.toGroupID, Equals, "explicit_group_id") c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.fromModuleID, Equals, fromMod.ID) c.Assert(ref.toModuleID, Equals, "module_id") c.Assert(ref.explicit, Equals, true) - ref, err = identifyModuleByReference(fmt.Sprintf("%s.module_id", dg.Name), dg) + ref, err = identifyModuleByReference(fmt.Sprintf("%s.module_id", dg.Name), dg, fromMod) c.Assert(err, IsNil) c.Assert(ref.toGroupID, Equals, dg.Name) c.Assert(ref.fromGroupID, Equals, dg.Name) + c.Assert(ref.fromModuleID, Equals, fromMod.ID) c.Assert(ref.toModuleID, Equals, "module_id") c.Assert(ref.explicit, Equals, true) - ref, err = identifyModuleByReference("explicit_group_id.module_id.output_name", dg) + ref, err = identifyModuleByReference("explicit_group_id.module_id.output_name", dg, fromMod) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["invalidMod"])) - ref, err = identifyModuleByReference("module_id.", dg) + ref, err = identifyModuleByReference("module_id.", dg, fromMod) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["invalidMod"])) - ref, err = identifyModuleByReference(".module_id", dg) + ref, err = identifyModuleByReference(".module_id", dg, fromMod) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: .*", errorMessages["invalidMod"])) } @@ -685,46 +692,53 @@ func (s *MySuite) TestIdentifySimpleVariable(c *C) { var err error dg := DeploymentGroup{ - Name: "calling_group_id", + Name: "from_group_id", } - ref, err = identifySimpleVariable("group_id.module_id.output_name", dg) + fromMod := Module{ + ID: "from_module_id", + } + + ref, err = identifySimpleVariable("group_id.module_id.output_name", dg, fromMod) c.Assert(err, IsNil) c.Assert(ref.toGroupID, Equals, "group_id") c.Assert(ref.fromGroupID, Equals, dg.Name) c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.fromModuleID, Equals, fromMod.ID) c.Assert(ref.name, Equals, "output_name") c.Assert(ref.explicit, Equals, true) - ref, err = identifySimpleVariable("module_id.output_name", dg) + ref, err = identifySimpleVariable("module_id.output_name", dg, fromMod) c.Assert(err, IsNil) - c.Assert(ref.toGroupID, Equals, "calling_group_id") - c.Assert(ref.fromGroupID, Equals, "calling_group_id") + c.Assert(ref.toGroupID, Equals, dg.Name) + c.Assert(ref.fromGroupID, Equals, dg.Name) c.Assert(ref.toModuleID, Equals, "module_id") + c.Assert(ref.fromModuleID, Equals, fromMod.ID) c.Assert(ref.name, Equals, "output_name") c.Assert(ref.explicit, Equals, false) - ref, err = identifySimpleVariable("vars.variable_name", dg) + ref, err = identifySimpleVariable("vars.variable_name", dg, fromMod) c.Assert(err, IsNil) c.Assert(ref.toGroupID, Equals, "deployment") c.Assert(ref.fromGroupID, Equals, dg.Name) c.Assert(ref.toModuleID, Equals, "vars") + c.Assert(ref.fromModuleID, Equals, fromMod.ID) c.Assert(ref.name, Equals, "variable_name") c.Assert(ref.explicit, Equals, false) - ref, err = identifySimpleVariable("foo", dg) + ref, err = identifySimpleVariable("foo", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable("foo.bar.baz.qux", dg) + ref, err = identifySimpleVariable("foo.bar.baz.qux", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable("foo..bar", dg) + ref, err = identifySimpleVariable("foo..bar", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable("foo.bar.", dg) + ref, err = identifySimpleVariable("foo.bar.", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable("foo..", dg) + ref, err = identifySimpleVariable("foo..", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable(".foo", dg) + ref, err = identifySimpleVariable(".foo", dg, fromMod) c.Assert(err, NotNil) - ref, err = identifySimpleVariable("..foo", dg) + ref, err = identifySimpleVariable("..foo", dg, fromMod) c.Assert(err, NotNil) } From b08dc376d5324782a181c083df89c01c3acceae6 Mon Sep 17 00:00:00 2001 From: Carlos Boneti Date: Wed, 29 Mar 2023 18:04:43 -0700 Subject: [PATCH 073/100] Updating requirements for ofe --- community/front-end/ofe/requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index ad67a9efbc..fdb49ae158 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -16,19 +16,19 @@ django-extensions==3.2.1 # Need version 0.11.0 to be released with fixes for Django 3.2 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 djangorestframework==3.13.1 -google-api-core==2.5.0 +google-api-core==2.11.0 google-api-python-client==2.82.0 -google-auth==2.6.0 +google-auth==2.16.2 google-auth-httplib2==0.1.0 -google-cloud-billing==1.4.1 +google-cloud-billing==1.10.0 google-cloud-core==2.3.2 -google-cloud-pubsub==2.9.0 -google-cloud-storage==2.1.0 -google-crc32c==1.3.0 -google-resumable-media==2.2.1 +google-cloud-pubsub==2.15.1 +google-cloud-storage==2.7.0 +google-crc32c==1.5.0 +google-resumable-media==2.4.1 googleapis-common-protos==1.58.0 grafana_api==1.0.3 -grpc-google-iam-v1==0.12.3 +grpc-google-iam-v1==0.12.6 grpcio==1.51.3 grpcio-status==1.51.3 h11==0.13.0 From ffeaafc77e8dedd68ec57cbb0e35b0716c3e4d96 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 12:10:27 +0100 Subject: [PATCH 074/100] Updating requirements.txt to the latest pip suggested --- community/front-end/ofe/requirements.txt | 103 +++++++++++++---------- 1 file changed, 58 insertions(+), 45 deletions(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index fdb49ae158..d9fb6b7756 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -1,72 +1,85 @@ -archspec==0.1.3 -argcomplete==2.0.0 -asgiref==3.5.0 -astroid==2.9.3 -cachetools==5.0.0 +archspec==0.2.0 +argcomplete==3.0.5 +asgiref==3.6.0 +astroid==2.15.1 +backports.zoneinfo==0.2.1 +cachetools==5.3.0 certifi==2022.12.7 -cffi==1.15.0 -charset-normalizer==2.0.12 -click==7.1.2 -cryptography==39.0.1 +cffi==1.15.1 +cfgv==3.3.1 +charset-normalizer==3.1.0 +click==8.1.3 +cryptography==40.0.1 +decorator==5.1.1 defusedxml==0.7.1 dill==0.3.6 +distlib==0.3.6 +# Can't go higher because of https://github.com/jazzband/django-revproxy/issues/144 Django==3.2.18 -django-allauth==0.48.0 -django-extensions==3.2.1 -# Need version 0.11.0 to be released with fixes for Django 3.2 -git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 -djangorestframework==3.13.1 +django-allauth==0.53.1 +django-extensions==3.1.5 +django-revproxy==0.10.0 +djangorestframework==3.14.0 +filelock==3.10.7 google-api-core==2.11.0 -google-api-python-client==2.82.0 -google-auth==2.16.2 +google-api-python-client==2.83.0 +google-auth==2.17.0 google-auth-httplib2==0.1.0 -google-cloud-billing==1.10.0 +google-cloud-billing==1.10.1 google-cloud-core==2.3.2 -google-cloud-pubsub==2.15.1 -google-cloud-storage==2.7.0 +google-cloud-pubsub==2.15.2 +google-cloud-storage==2.8.0 google-crc32c==1.5.0 google-resumable-media==2.4.1 -googleapis-common-protos==1.58.0 -grafana_api==1.0.3 +googleapis-common-protos==1.59.0 +grafana-api==1.0.3 grpc-google-iam-v1==0.12.6 -grpcio==1.51.3 -grpcio-status==1.51.3 -h11==0.13.0 -httplib2==0.21.0 -idna==3.3 +grpcio==1.53.0 +grpcio-status==1.53.0 +h11==0.14.0 +httplib2==0.22.0 +identify==2.5.22 +idna==3.4 isort==5.12.0 lazy-object-proxy==1.9.0 -libcst==0.4.1 -mccabe==0.6.1 -mypy-extensions==0.4.3 +libcst==0.4.9 +mccabe==0.7.0 +mypy-extensions==1.0.0 +nodeenv==1.7.0 oauthlib==3.2.2 -platformdirs==3.1.1 -pre-commit==2.17.0 +platformdirs==3.2.0 +pre-commit==3.2.1 proto-plus==1.22.2 protobuf==4.22.1 +py==1.11.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 PyJWT==2.6.0 -pylint==2.12.2 -pylint-django==2.5.0 +pylint==2.17.1 +pylint-django==2.5.3 pylint-plugin-utils==0.7 pyparsing==3.0.9 python3-openid==3.2.0 -pytz==2021.3 +pytz==2023.3 PyYAML==6.0 -retry==0.9.2 -requests==2.27.1 +requests==2.28.2 requests-oauthlib==1.3.1 -rsa==4.8 +retry==0.9.2 +rsa==4.9 +semantic-version==2.10.0 +setuptools-rust==1.5.2 six==1.16.0 -sqlparse==0.4.2 +sqlparse==0.4.3 toml==0.10.2 -typing-inspect==0.7.1 -typing_extensions==4.1.1 +tomli==2.0.1 +tomlkit==0.11.7 +typing-inspect==0.8.0 +typing_extensions==4.5.0 uritemplate==4.1.1 -urllib3==1.26.8 -uvicorn==0.17.4 -wrapt==1.13.3 -xmltodict==0.12.0 -yq==2.13.0 +urllib3==1.26.15 +uvicorn==0.21.1 +virtualenv==20.21.0 +wrapt==1.15.0 +xmltodict==0.13.0 +yq==3.1.1 From 1d97eb12e90d34d81f0eb62de4b07a67c1f75799 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 12:17:08 +0100 Subject: [PATCH 075/100] New Django versions require app name in makemigrations --- .../ofe/infrastructure_files/gcs_bucket/webserver/startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh b/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh index 0e003125c0..47c1ac2cd2 100644 --- a/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh +++ b/community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh @@ -192,7 +192,7 @@ sudo su - gcluster -c /bin/bash < Date: Fri, 31 Mar 2023 12:18:17 +0100 Subject: [PATCH 076/100] fixing issue: #1096 --- .../front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py b/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py index 184b89ca28..33d09312bd 100644 --- a/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py +++ b/community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py @@ -56,7 +56,7 @@ def write_filestore_yaml(fs: GCPFilestoreFilesystem, target_dir: Path) -> None: id: {fs.name} settings: filestore_share_name: {export_name[1:]} - network_name: {fs.vpc.cloud_id} + network_id: projects/{project_id}/global/networks/{fs.vpc.cloud_id} zone: {fs.cloud_zone} size_gb: {fs.capacity} filestore_tier: {fs.get_performance_tier_display()} From 16161b4aed310345d1007e09e2125c580bc92c21 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 12:19:11 +0100 Subject: [PATCH 077/100] community/front-end/ofe/website/ghpcfe/views/grafana.py --- community/front-end/ofe/website/ghpcfe/views/grafana.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/website/ghpcfe/views/grafana.py b/community/front-end/ofe/website/ghpcfe/views/grafana.py index dc92c47b16..6d9f05094d 100644 --- a/community/front-end/ofe/website/ghpcfe/views/grafana.py +++ b/community/front-end/ofe/website/ghpcfe/views/grafana.py @@ -29,7 +29,7 @@ def get_proxy_request_headers(self, request): def dispatch(self, request, path): response = super().dispatch(request, path) - response.headers["X-Frame-Options"] = "SAMEORIGIN" + response["X-Frame-Options"] = "SAMEORIGIN" return response class GrafanaView(LoginRequiredMixin, base.TemplateView): From d7be6cc096a44e4cd02aca130a499771f855a401 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 13:59:42 +0100 Subject: [PATCH 078/100] django-revproxy release 0.11.0 allows higher Django version --- community/front-end/ofe/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index d9fb6b7756..5218589dec 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -14,11 +14,11 @@ decorator==5.1.1 defusedxml==0.7.1 dill==0.3.6 distlib==0.3.6 -# Can't go higher because of https://github.com/jazzband/django-revproxy/issues/144 -Django==3.2.18 +# django-revproxy==0.11.0 released but not yet in pypi +git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 +Django==4.1.7 django-allauth==0.53.1 django-extensions==3.1.5 -django-revproxy==0.10.0 djangorestframework==3.14.0 filelock==3.10.7 google-api-core==2.11.0 From cbfbf979daf4635acd8722aad488a236de732ea5 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 14:20:33 +0100 Subject: [PATCH 079/100] To access VPC subnets by vpc.id, vpc must be firstly created --- community/front-end/ofe/website/ghpcfe/signals.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/community/front-end/ofe/website/ghpcfe/signals.py b/community/front-end/ofe/website/ghpcfe/signals.py index a51f4ae27b..fc14cbc040 100644 --- a/community/front-end/ofe/website/ghpcfe/signals.py +++ b/community/front-end/ofe/website/ghpcfe/signals.py @@ -14,14 +14,14 @@ """Signal handlers for model state""" -from django.db.models.signals import pre_save, post_delete +from django.db.models.signals import pre_save, post_delete, post_save from django.dispatch import receiver from .models import Cluster, VirtualNetwork # Pylint misses the sender decorator behaviour here #pylint: disable=unused-argument -@receiver(pre_save, sender=VirtualNetwork) +@receiver(post_save, sender=VirtualNetwork) def sync_vnet_subnet_state(sender, **kwargs): vpc = kwargs["instance"] for sn in vpc.subnets.all(): From fa6c324c38abcb639d593fe6b8cf4d30786aac14 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 15:17:01 +0100 Subject: [PATCH 080/100] newer Grafana needs Host header in request --- community/front-end/ofe/website/ghpcfe/views/grafana.py | 1 + 1 file changed, 1 insertion(+) diff --git a/community/front-end/ofe/website/ghpcfe/views/grafana.py b/community/front-end/ofe/website/ghpcfe/views/grafana.py index 6d9f05094d..6dde8559c6 100644 --- a/community/front-end/ofe/website/ghpcfe/views/grafana.py +++ b/community/front-end/ofe/website/ghpcfe/views/grafana.py @@ -25,6 +25,7 @@ class GrafanaProxyView(LoginRequiredMixin, ProxyView): def get_proxy_request_headers(self, request): headers = super().get_proxy_request_headers(request) headers["X-WEBAUTH-USER"] = request.user.email + headers["Host"] = request.get_host() return headers def dispatch(self, request, path): From efd333b56d33039f4411ffa648077c4a70c44c19 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 09:38:49 -0500 Subject: [PATCH 081/100] Fix IsIntergroup return value --- pkg/config/expand.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 4b122e09ae..87ad08d122 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -538,7 +538,7 @@ func (ref modReference) String() string { } func (ref modReference) IsIntergroup() bool { - return ref.toGroupID == ref.fromGroupID + return ref.toGroupID != ref.fromGroupID } func (ref modReference) FromModuleID() string { @@ -610,7 +610,7 @@ func (ref varReference) String() string { } func (ref varReference) IsIntergroup() bool { - return ref.toGroupID == ref.fromGroupID + return ref.toGroupID != ref.fromGroupID } func (ref varReference) FromModuleID() string { From b040027f16093b93ec0a276841d1ea4250b86a3f Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 09:38:50 -0500 Subject: [PATCH 082/100] Track usage of deployment variables by modules --- pkg/config/config.go | 6 +-- pkg/config/config_test.go | 110 ++++++++++++++++++++++++++++++++++---- pkg/config/expand.go | 40 ++++++++------ pkg/config/expand_test.go | 10 ++-- 4 files changed, 136 insertions(+), 30 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 51ec1ce8fa..35b8221e32 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,7 +38,7 @@ import ( const ( expectedVarFormat string = "$(vars.var_name) or $(module_id.output_name)" expectedModFormat string = "$(module_id) or $(group_id.module_id)" - unexpectedConnectionKind string = "connectionKind must be useConnection" + unexpectedConnectionKind string = "connectionKind must be useConnection or deploymentConnection" ) var errorMessages = map[string]string{ @@ -267,12 +267,12 @@ type connectionKind int const ( undefinedConnection connectionKind = iota useConnection + deploymentConnection // explicitConnection - // globalConnection ) func (c connectionKind) IsValid() bool { - return c == useConnection + return c == useConnection || c == deploymentConnection } // ModConnection defines details about connections between modules. Currently, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 29aad784e8..4f9f357153 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,6 +30,8 @@ import ( "github.com/pkg/errors" "github.com/zclconf/go-cty/cty" + "golang.org/x/exp/maps" + "golang.org/x/exp/slices" . "gopkg.in/check.v1" ) @@ -259,7 +261,16 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { matchingName := "test_match" testModuleInfo0 := modulereader.ModuleInfo{ - Inputs: []modulereader.VarInfo{}, + Inputs: []modulereader.VarInfo{ + { + Name: "deployment_name", + Type: "string", + }, + { + Name: "host_project_id", + Type: "string", + }, + }, Outputs: []modulereader.VarInfo{ { Name: matchingName, @@ -268,6 +279,10 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { } testModuleInfo1 := modulereader.ModuleInfo{ Inputs: []modulereader.VarInfo{ + { + Name: "deployment_name", + Type: "string", + }, { Name: matchingName, }, @@ -279,11 +294,13 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { Name: "primary", Modules: []Module{ { - ID: "TestModule0", - Kind: "terraform", - Source: testModuleSource0, - Settings: map[string]interface{}{}, - Outputs: []string{matchingName}, + ID: "TestModule0", + Kind: "terraform", + Source: testModuleSource0, + Settings: map[string]interface{}{ + "host_project_id": "$(vars.project_id)", + }, + Outputs: []string{matchingName}, }, }, } @@ -303,7 +320,15 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { } dc := DeploymentConfig{ - Config: Blueprint{BlueprintName: "simple", Vars: map[string]interface{}{"deployment_name": "deployment_name"}, DeploymentGroups: []DeploymentGroup{testDeploymentGroup0, testDeploymentGroup1}}, + Config: Blueprint{ + BlueprintName: "simple", + Vars: map[string]interface{}{ + "deployment_name": "deployment_name", + "project_id": "test-project", + "unused_key": "unused_value", + }, + DeploymentGroups: []DeploymentGroup{testDeploymentGroup0, testDeploymentGroup1}, + }, ModulesInfo: map[string]map[string]modulereader.ModuleInfo{ testDeploymentGroup0.Name: { testModuleSource0: testModuleInfo0, @@ -315,8 +340,10 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { moduleConnections: make(map[string][]ModConnection), } - dc.addMetadataToModules() - dc.addDefaultValidators() + reader := modulereader.Factory("terraform") + reader.SetInfo(testModuleSource0, testModuleInfo0) + reader.SetInfo(testModuleSource1, testModuleInfo1) + return dc } @@ -473,6 +500,71 @@ func (s *MySuite) TestAddKindToModules(c *C) { c.Assert(testMod.Kind, Equals, expected) } +func (s *MySuite) TestModuleConnections(c *C) { + dc := getMultiGroupDeploymentConfig() + modID0 := dc.Config.DeploymentGroups[0].Modules[0].ID + modID1 := dc.Config.DeploymentGroups[1].Modules[0].ID + + err := dc.applyUseModules() + c.Assert(err, IsNil) + err = dc.applyGlobalVariables() + c.Assert(err, IsNil) + err = dc.expandVariables() + // TODO: this will become nil once intergroup references are enabled + c.Assert(err, NotNil) + + // check that ModuleConnections has map keys for each module ID + c.Assert(len(dc.moduleConnections), Equals, 2) + connectionIDs := maps.Keys(dc.moduleConnections) + found := slices.Contains(connectionIDs, modID0) + c.Assert(found, Equals, true) + found = slices.Contains(connectionIDs, modID1) + c.Assert(found, Equals, true) + + // check connections for module 0 (only to deployment variables) + connectionsMod0 := dc.moduleConnections[modID0] + c.Assert(len(connectionsMod0), Equals, 2) + found = slices.ContainsFunc(connectionsMod0, func(c ModConnection) bool { + return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"project_id"}) + }) + c.Assert(found, Equals, true) + found = slices.ContainsFunc(connectionsMod0, func(c ModConnection) bool { + return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"deployment_name"}) + }) + c.Assert(found, Equals, true) + + // check connections for module 1 (1 deployment variable and 1 intergroup use) + connectionsMod1 := dc.moduleConnections[modID1] + c.Assert(len(connectionsMod1), Equals, 2) + found = slices.ContainsFunc(connectionsMod1, func(c ModConnection) bool { + return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"deployment_name"}) + }) + c.Assert(found, Equals, true) + found = slices.ContainsFunc(connectionsMod1, func(c ModConnection) bool { + return c.kind == useConnection && + slices.Equal(c.sharedVariables, []string{"test_match"}) && + c.ref.IsIntergroup() + }) + c.Assert(found, Equals, true) + + // the method of applying $(vars.project_id) to required APIs and default + // validators creates 2 extra edges in the graph that are attributed to group + // 0 and module 0 (L962 and L980 of expand.go) + dc = getMultiGroupDeploymentConfig() + dc.addSettingsToModules() + dc.addMetadataToModules() + dc.addDefaultValidators() + err = dc.applyUseModules() + c.Assert(err, IsNil) + err = dc.applyGlobalVariables() + c.Assert(err, IsNil) + err = dc.expandVariables() + // TODO: this will become nil once intergroup references are enabled + c.Assert(err, NotNil) + connectionsMod0 = dc.moduleConnections[modID0] + c.Assert(len(connectionsMod0), Equals, 4) +} + func (s *MySuite) TestSetModulesInfo(c *C) { dc := getBasicDeploymentConfigWithTestModule() dc.setModulesInfo() diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 87ad08d122..e5c631cd61 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -441,10 +441,11 @@ func mergeInLabels[V interface{}](to map[string]V, from map[string]V) { } } -func applyGlobalVarsInGroup( - deploymentGroup DeploymentGroup, - modInfo map[string]modulereader.ModuleInfo, - globalVars map[string]interface{}) error { +func (dc DeploymentConfig) applyGlobalVarsInGroup(groupIndex int) error { + deploymentGroup := dc.Config.DeploymentGroups[groupIndex] + modInfo := dc.ModulesInfo[deploymentGroup.Name] + globalVars := dc.Config.Vars + for _, mod := range deploymentGroup.Modules { for _, input := range modInfo[mod.Source].Inputs { @@ -455,7 +456,16 @@ func applyGlobalVarsInGroup( // If it's not set, is there a global we can use? if _, ok := globalVars[input.Name]; ok { + ref := varReference{ + name: input.Name, + toModuleID: "vars", + fromModuleID: mod.ID, + toGroupID: "deployment", + fromGroupID: deploymentGroup.Name, + explicit: false, + } mod.Settings[input.Name] = fmt.Sprintf("((var.%s))", input.Name) + dc.addModuleConnection(ref, deploymentConnection, []string{ref.name}) continue } @@ -490,9 +500,8 @@ func (dc *DeploymentConfig) applyGlobalVariables() error { return err } - for _, grp := range dc.Config.DeploymentGroups { - err := applyGlobalVarsInGroup( - grp, dc.ModulesInfo[grp.Name], dc.Config.Vars) + for groupIndex := range dc.Config.DeploymentGroups { + err := dc.applyGlobalVarsInGroup(groupIndex) if err != nil { return err } @@ -504,7 +513,7 @@ type varContext struct { varString string groupIndex int modIndex int - blueprint Blueprint + dc DeploymentConfig } type reference interface { @@ -797,7 +806,7 @@ func expandSimpleVariable(context varContext) (string, error) { return "", err } - callingGroup := context.blueprint.DeploymentGroups[context.groupIndex] + callingGroup := context.dc.Config.DeploymentGroups[context.groupIndex] refStr := contents[1] varRef, err := identifySimpleVariable(refStr, callingGroup, callingGroup.Modules[context.modIndex]) @@ -805,7 +814,7 @@ func expandSimpleVariable(context varContext) (string, error) { return "", err } - if err := varRef.validate(context.blueprint); err != nil { + if err := varRef.validate(context.dc.Config); err != nil { return "", err } @@ -814,6 +823,7 @@ func expandSimpleVariable(context varContext) (string, error) { case "deployment": // deployment variables expandedVariable = fmt.Sprintf("((var.%s))", varRef.name) + context.dc.addModuleConnection(varRef, deploymentConnection, []string{varRef.name}) case varRef.fromGroupID: // intragroup reference can make direct reference to module output expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.toModuleID, varRef.name) @@ -821,13 +831,13 @@ func expandSimpleVariable(context varContext) (string, error) { // intergroup reference; begin by finding the target module in blueprint toGrpIdx := slices.IndexFunc( - context.blueprint.DeploymentGroups, + context.dc.Config.DeploymentGroups, func(g DeploymentGroup) bool { return g.Name == varRef.toGroupID }) if toGrpIdx == -1 { return "", fmt.Errorf("invalid group reference: %s", varRef.toGroupID) } - toGrp := context.blueprint.DeploymentGroups[toGrpIdx] + toGrp := context.dc.Config.DeploymentGroups[toGrpIdx] toModIdx := slices.IndexFunc(toGrp.Modules, func(m Module) bool { return m.ID == varRef.toModuleID }) if toModIdx == -1 { return "", fmt.Errorf("%s: %s", errorMessages["invalidMod"], varRef.toModuleID) @@ -947,7 +957,7 @@ func updateVariables(context varContext, interfaceMap map[string]interface{}) er // expands all variables func (dc *DeploymentConfig) expandVariables() error { for _, validator := range dc.Config.Validators { - err := updateVariables(varContext{blueprint: dc.Config}, validator.Inputs) + err := updateVariables(varContext{dc: *dc}, validator.Inputs) if err != nil { return err @@ -959,7 +969,7 @@ func (dc *DeploymentConfig) expandVariables() error { context := varContext{ groupIndex: iGrp, modIndex: iMod, - blueprint: dc.Config, + dc: *dc, } err := updateVariables(context, mod.Settings) if err != nil { @@ -969,7 +979,7 @@ func (dc *DeploymentConfig) expandVariables() error { // ensure that variable references to projects in required APIs are expanded for projectID, requiredAPIs := range mod.RequiredApis { if isDeploymentVariable(projectID) { - s, err := handleVariable(projectID, varContext{blueprint: dc.Config}) + s, err := handleVariable(projectID, context) if err != nil { return err } diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 865647bd93..45881d6948 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -773,13 +773,17 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { } testVarContext0 := varContext{ - blueprint: testBlueprint, + dc: DeploymentConfig{ + Config: testBlueprint, + }, modIndex: 0, groupIndex: 0, } testVarContext1 := varContext{ - blueprint: testBlueprint, + dc: DeploymentConfig{ + Config: testBlueprint, + }, modIndex: 0, groupIndex: 1, } @@ -797,7 +801,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { c.Assert(err, ErrorMatches, expectedErr) // Global variable: Success - testVarContext1.blueprint.Vars["globalExists"] = "existsValue" + testVarContext1.dc.Config.Vars["globalExists"] = "existsValue" testVarContext1.varString = "$(vars.globalExists)" got, err := expandSimpleVariable(testVarContext1) c.Assert(err, IsNil) From c8302b72f76695bfdd4ad38842a0bbc224327a1d Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 09:38:50 -0500 Subject: [PATCH 083/100] Ensure that graph edges are only tracked for module connections --- pkg/config/config_test.go | 21 ++++-------------- pkg/config/expand.go | 38 +++++++++++++++++--------------- pkg/config/expand_test.go | 46 +++++++++++++++++++-------------------- 3 files changed, 48 insertions(+), 57 deletions(-) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 4f9f357153..1bcb21a174 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -505,6 +505,10 @@ func (s *MySuite) TestModuleConnections(c *C) { modID0 := dc.Config.DeploymentGroups[0].Modules[0].ID modID1 := dc.Config.DeploymentGroups[1].Modules[0].ID + dc.addSettingsToModules() + dc.addMetadataToModules() + dc.addDefaultValidators() + err := dc.applyUseModules() c.Assert(err, IsNil) err = dc.applyGlobalVariables() @@ -546,23 +550,6 @@ func (s *MySuite) TestModuleConnections(c *C) { c.ref.IsIntergroup() }) c.Assert(found, Equals, true) - - // the method of applying $(vars.project_id) to required APIs and default - // validators creates 2 extra edges in the graph that are attributed to group - // 0 and module 0 (L962 and L980 of expand.go) - dc = getMultiGroupDeploymentConfig() - dc.addSettingsToModules() - dc.addMetadataToModules() - dc.addDefaultValidators() - err = dc.applyUseModules() - c.Assert(err, IsNil) - err = dc.applyGlobalVariables() - c.Assert(err, IsNil) - err = dc.expandVariables() - // TODO: this will become nil once intergroup references are enabled - c.Assert(err, NotNil) - connectionsMod0 = dc.moduleConnections[modID0] - c.Assert(len(connectionsMod0), Equals, 4) } func (s *MySuite) TestSetModulesInfo(c *C) { diff --git a/pkg/config/expand.go b/pkg/config/expand.go index e5c631cd61..23dfa10459 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -483,7 +483,7 @@ func (dc DeploymentConfig) applyGlobalVarsInGroup(groupIndex int) error { func updateGlobalVarTypes(vars map[string]interface{}) error { for k, v := range vars { - val, err := updateVariableType(v, varContext{}) + val, err := updateVariableType(v, varContext{}, false) if err != nil { return fmt.Errorf("error setting type for deployment variable %s: %v", k, err) } @@ -797,7 +797,7 @@ func (ref varReference) validate(bp Blueprint) error { } // Needs DeploymentGroups, variable string, current group, -func expandSimpleVariable(context varContext) (string, error) { +func expandSimpleVariable(context varContext, trackModuleGraph bool) (string, error) { // Get variable contents contents := simpleVariableExp.FindStringSubmatch(context.varString) if len(contents) != 2 { // Should always be (match, contents) here @@ -823,7 +823,9 @@ func expandSimpleVariable(context varContext) (string, error) { case "deployment": // deployment variables expandedVariable = fmt.Sprintf("((var.%s))", varRef.name) - context.dc.addModuleConnection(varRef, deploymentConnection, []string{varRef.name}) + if trackModuleGraph { + context.dc.addModuleConnection(varRef, deploymentConnection, []string{varRef.name}) + } case varRef.fromGroupID: // intragroup reference can make direct reference to module output expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.toModuleID, varRef.name) @@ -857,7 +859,9 @@ func expandSimpleVariable(context varContext) (string, error) { return expandedVariable, nil } -func expandVariable(context varContext) (string, error) { +// expandVariable will eventually implement string interpolation; right now it +// only generates an error message guiding the user to proper escape syntax +func expandVariable(context varContext, trackModuleGraph bool) (string, error) { matchall := anyVariableExp.FindAllString(context.varString, -1) errHint := "" for _, element := range matchall { @@ -889,15 +893,15 @@ func hasVariable(str string) bool { return anyVariableExp.MatchString(str) } -func handleVariable(prim interface{}, context varContext) (interface{}, error) { +func handleVariable(prim interface{}, context varContext, trackModuleGraph bool) (interface{}, error) { switch val := prim.(type) { case string: context.varString = val if hasVariable(val) { if isSimpleVariable(val) { - return expandSimpleVariable(context) + return expandSimpleVariable(context, trackModuleGraph) } - return expandVariable(context) + return expandVariable(context, trackModuleGraph) } return val, nil default: @@ -905,14 +909,14 @@ func handleVariable(prim interface{}, context varContext) (interface{}, error) { } } -func updateVariableType(value interface{}, context varContext) (interface{}, error) { +func updateVariableType(value interface{}, context varContext, trackModuleGraph bool) (interface{}, error) { var err error switch typedValue := value.(type) { case []interface{}: interfaceSlice := value.([]interface{}) { for i := 0; i < len(interfaceSlice); i++ { - interfaceSlice[i], err = updateVariableType(interfaceSlice[i], context) + interfaceSlice[i], err = updateVariableType(interfaceSlice[i], context, trackModuleGraph) if err != nil { return interfaceSlice, err } @@ -922,7 +926,7 @@ func updateVariableType(value interface{}, context varContext) (interface{}, err case map[string]interface{}: retMap := map[string]interface{}{} for k, v := range typedValue { - retMap[k], err = updateVariableType(v, context) + retMap[k], err = updateVariableType(v, context, trackModuleGraph) if err != nil { return retMap, err } @@ -931,20 +935,20 @@ func updateVariableType(value interface{}, context varContext) (interface{}, err case map[interface{}]interface{}: retMap := map[string]interface{}{} for k, v := range typedValue { - retMap[k.(string)], err = updateVariableType(v, context) + retMap[k.(string)], err = updateVariableType(v, context, trackModuleGraph) if err != nil { return retMap, err } } return retMap, err default: - return handleVariable(value, context) + return handleVariable(value, context, trackModuleGraph) } } -func updateVariables(context varContext, interfaceMap map[string]interface{}) error { +func updateVariables(context varContext, interfaceMap map[string]interface{}, trackModuleGraph bool) error { for key, value := range interfaceMap { - updatedVal, err := updateVariableType(value, context) + updatedVal, err := updateVariableType(value, context, trackModuleGraph) if err != nil { return err } @@ -957,7 +961,7 @@ func updateVariables(context varContext, interfaceMap map[string]interface{}) er // expands all variables func (dc *DeploymentConfig) expandVariables() error { for _, validator := range dc.Config.Validators { - err := updateVariables(varContext{dc: *dc}, validator.Inputs) + err := updateVariables(varContext{dc: *dc}, validator.Inputs, false) if err != nil { return err @@ -971,7 +975,7 @@ func (dc *DeploymentConfig) expandVariables() error { modIndex: iMod, dc: *dc, } - err := updateVariables(context, mod.Settings) + err := updateVariables(context, mod.Settings, true) if err != nil { return err } @@ -979,7 +983,7 @@ func (dc *DeploymentConfig) expandVariables() error { // ensure that variable references to projects in required APIs are expanded for projectID, requiredAPIs := range mod.RequiredApis { if isDeploymentVariable(projectID) { - s, err := handleVariable(projectID, context) + s, err := handleVariable(projectID, context, false) if err != nil { return err } diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 45881d6948..7030f322d7 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -280,49 +280,49 @@ func (s *MySuite) TestUpdateVariableType(c *C) { // empty testSlice := []interface{}{} ctx := varContext{} - ret, err := updateVariableType(testSlice, ctx) + ret, err := updateVariableType(testSlice, ctx, false) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // single string testSlice = append(testSlice, "string") - ret, err = updateVariableType(testSlice, ctx) + ret, err = updateVariableType(testSlice, ctx, false) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // add list testSlice = append(testSlice, []interface{}{}) - ret, err = updateVariableType(testSlice, ctx) + ret, err = updateVariableType(testSlice, ctx, false) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // add map testSlice = append(testSlice, make(map[string]interface{})) - ret, err = updateVariableType(testSlice, ctx) + ret, err = updateVariableType(testSlice, ctx, false) c.Assert(err, IsNil) c.Assert(testSlice, DeepEquals, ret) // map, success testMap := make(map[string]interface{}) - ret, err = updateVariableType(testMap, ctx) + ret, err = updateVariableType(testMap, ctx, false) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add string testMap["string"] = "string" - ret, err = updateVariableType(testMap, ctx) + ret, err = updateVariableType(testMap, ctx, false) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add map testMap["map"] = make(map[string]interface{}) - ret, err = updateVariableType(testMap, ctx) + ret, err = updateVariableType(testMap, ctx, false) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // add slice testMap["slice"] = []interface{}{} - ret, err = updateVariableType(testMap, ctx) + ret, err = updateVariableType(testMap, ctx, false) c.Assert(err, IsNil) c.Assert(testMap, DeepEquals, ret) // string, success testString := "string" - ret, err = updateVariableType(testString, ctx) + ret, err = updateVariableType(testString, ctx, false) c.Assert(err, IsNil) c.Assert(testString, DeepEquals, ret) } @@ -790,26 +790,26 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { // Invalid variable -> no . testVarContext1.varString = "$(varsStringWithNoDot)" - _, err := expandSimpleVariable(testVarContext1) + _, err := expandSimpleVariable(testVarContext1, false) expectedErr := fmt.Sprintf("%s.*", errorMessages["invalidVar"]) c.Assert(err, ErrorMatches, expectedErr) // Global variable: Invalid -> not found testVarContext1.varString = "$(vars.doesntExists)" - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) expectedErr = fmt.Sprintf("%s: .*", errorMessages["varNotFound"]) c.Assert(err, ErrorMatches, expectedErr) // Global variable: Success testVarContext1.dc.Config.Vars["globalExists"] = "existsValue" testVarContext1.varString = "$(vars.globalExists)" - got, err := expandSimpleVariable(testVarContext1) + got, err := expandSimpleVariable(testVarContext1, false) c.Assert(err, IsNil) c.Assert(got, Equals, "((var.globalExists))") // Module variable: Invalid -> Module not found testVarContext1.varString = "$(bad_mod.someVar)" - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, ErrorMatches, "module bad_mod was not found") // Module variable: Invalid -> Output not found @@ -817,7 +817,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, modulereader.ModuleInfo{}) fakeOutput := "doesntExist" testVarContext1.varString = fmt.Sprintf("$(%s.%s)", testModule1.ID, fakeOutput) - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) expectedErr = fmt.Sprintf("%s: module %s did not have output %s", errorMessages["noOutput"], testModule1.ID, fakeOutput) c.Assert(err, ErrorMatches, expectedErr) @@ -831,7 +831,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s)", testModule1.ID, existingOutput) - got, err = expandSimpleVariable(testVarContext1) + got, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, IsNil) expectedErr = fmt.Sprintf("((module.%s.%s))", testModule1.ID, existingOutput) c.Assert(got, Equals, expectedErr) @@ -845,7 +845,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[1].Name, testModule1.ID, existingOutput) - got, err = expandSimpleVariable(testVarContext1) + got, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, IsNil) c.Assert(got, Equals, fmt.Sprintf("((module.%s.%s))", testModule1.ID, existingOutput)) @@ -859,7 +859,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule1.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, NotNil) expectedErr = fmt.Sprintf("%s: %s.%s should be %s.%s", @@ -876,7 +876,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule0.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s)", testModule0.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) expectedErr = fmt.Sprintf("%s: %s .*", errorMessages["intergroupImplicit"], testModule0.ID) c.Assert(err, ErrorMatches, expectedErr) @@ -884,14 +884,14 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { // Intergroup variable: failure because explicit group and module does not exist testVarContext1.varString = fmt.Sprintf("$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, "bad_module", "bad_output") - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, ErrorMatches, "module bad_module was not found") // Intergroup variable: failure because explicit group and output does not exist fakeOutput = "bad_output" testVarContext1.varString = fmt.Sprintf("$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule0.ID, fakeOutput) - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) expectedErr = fmt.Sprintf("%s: module %s did not have output %s", errorMessages["noOutput"], testModule0.ID, fakeOutput) c.Assert(err, ErrorMatches, expectedErr) @@ -904,7 +904,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule1.Source, testModInfo) testVarContext0.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[1].Name, testModule1.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext0) + _, err = expandSimpleVariable(testVarContext0, false) expectedErr = fmt.Sprintf("%s: %s .*", errorMessages["intergroupOrder"], testModule1.ID) c.Assert(err, ErrorMatches, expectedErr) @@ -918,7 +918,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { reader.SetInfo(testModule0.Source, testModInfo) testVarContext1.varString = fmt.Sprintf( "$(%s.%s.%s)", testBlueprint.DeploymentGroups[0].Name, testModule0.ID, existingOutput) - _, err = expandSimpleVariable(testVarContext1) + _, err = expandSimpleVariable(testVarContext1, false) c.Assert(err, ErrorMatches, fmt.Sprintf("%s: %s .*", errorMessages["varInAnotherGroup"], regexp.QuoteMeta(testVarContext1.varString))) } @@ -926,7 +926,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { // presently it returns only an error func (s *MySuite) TestExpandVariable(c *C) { testVarContext0 := varContext{} - str, err := expandVariable(testVarContext0) + str, err := expandVariable(testVarContext0, false) c.Assert(str, Equals, "") c.Assert(err, NotNil) } From ab8a56ba3b43197d115200a9dbd3fabad3bdfae4 Mon Sep 17 00:00:00 2001 From: Eimantas Kazakevicius Date: Fri, 31 Mar 2023 15:53:54 +0100 Subject: [PATCH 084/100] removing unnecessary py package --- community/front-end/ofe/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 5218589dec..1cc9cdbcce 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -51,7 +51,6 @@ platformdirs==3.2.0 pre-commit==3.2.1 proto-plus==1.22.2 protobuf==4.22.1 -py==1.11.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 From ea656f9679418444de6c740de3536c987d1875e6 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 17:04:05 -0500 Subject: [PATCH 085/100] Address feedback from #1094 --- pkg/config/config_test.go | 85 ++++++++++++++++++++++++--------------- pkg/config/expand.go | 8 ++-- pkg/config/expand_test.go | 4 +- 3 files changed, 58 insertions(+), 39 deletions(-) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 1bcb21a174..f2612a4664 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -30,8 +30,6 @@ import ( "github.com/pkg/errors" "github.com/zclconf/go-cty/cty" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" . "gopkg.in/check.v1" ) @@ -518,38 +516,59 @@ func (s *MySuite) TestModuleConnections(c *C) { c.Assert(err, NotNil) // check that ModuleConnections has map keys for each module ID - c.Assert(len(dc.moduleConnections), Equals, 2) - connectionIDs := maps.Keys(dc.moduleConnections) - found := slices.Contains(connectionIDs, modID0) - c.Assert(found, Equals, true) - found = slices.Contains(connectionIDs, modID1) - c.Assert(found, Equals, true) - - // check connections for module 0 (only to deployment variables) - connectionsMod0 := dc.moduleConnections[modID0] - c.Assert(len(connectionsMod0), Equals, 2) - found = slices.ContainsFunc(connectionsMod0, func(c ModConnection) bool { - return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"project_id"}) - }) - c.Assert(found, Equals, true) - found = slices.ContainsFunc(connectionsMod0, func(c ModConnection) bool { - return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"deployment_name"}) - }) - c.Assert(found, Equals, true) - - // check connections for module 1 (1 deployment variable and 1 intergroup use) - connectionsMod1 := dc.moduleConnections[modID1] - c.Assert(len(connectionsMod1), Equals, 2) - found = slices.ContainsFunc(connectionsMod1, func(c ModConnection) bool { - return c.kind == deploymentConnection && slices.Equal(c.sharedVariables, []string{"deployment_name"}) - }) - c.Assert(found, Equals, true) - found = slices.ContainsFunc(connectionsMod1, func(c ModConnection) bool { - return c.kind == useConnection && - slices.Equal(c.sharedVariables, []string{"test_match"}) && - c.ref.IsIntergroup() + c.Check(dc.moduleConnections, DeepEquals, map[string][]ModConnection{ + modID0: { + { + ref: varReference{ + name: "deployment_name", + toModuleID: "vars", + fromModuleID: "TestModule0", + toGroupID: "deployment", + fromGroupID: "primary", + explicit: false, + }, + kind: deploymentConnection, + sharedVariables: []string{"deployment_name"}, + }, + { + ref: varReference{ + name: "project_id", + toModuleID: "vars", + fromModuleID: "TestModule0", + toGroupID: "deployment", + fromGroupID: "primary", + explicit: false, + }, + kind: deploymentConnection, + sharedVariables: []string{"project_id"}, + }, + }, + modID1: { + { + ref: modReference{ + toModuleID: "TestModule0", + fromModuleID: "TestModule1", + toGroupID: "primary", + fromGroupID: "secondary", + explicit: true, + }, + kind: useConnection, + sharedVariables: []string{"test_match"}, + }, + { + ref: varReference{ + name: "deployment_name", + toModuleID: "vars", + fromModuleID: "TestModule1", + toGroupID: "deployment", + fromGroupID: "secondary", + explicit: false, + }, + kind: deploymentConnection, + sharedVariables: []string{"deployment_name"}, + }, + }, }) - c.Assert(found, Equals, true) } func (s *MySuite) TestSetModulesInfo(c *C) { diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 23dfa10459..2240985804 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -441,7 +441,7 @@ func mergeInLabels[V interface{}](to map[string]V, from map[string]V) { } } -func (dc DeploymentConfig) applyGlobalVarsInGroup(groupIndex int) error { +func (dc *DeploymentConfig) applyGlobalVarsInGroup(groupIndex int) error { deploymentGroup := dc.Config.DeploymentGroups[groupIndex] modInfo := dc.ModulesInfo[deploymentGroup.Name] globalVars := dc.Config.Vars @@ -513,7 +513,7 @@ type varContext struct { varString string groupIndex int modIndex int - dc DeploymentConfig + dc *DeploymentConfig } type reference interface { @@ -961,7 +961,7 @@ func updateVariables(context varContext, interfaceMap map[string]interface{}, tr // expands all variables func (dc *DeploymentConfig) expandVariables() error { for _, validator := range dc.Config.Validators { - err := updateVariables(varContext{dc: *dc}, validator.Inputs, false) + err := updateVariables(varContext{dc: dc}, validator.Inputs, false) if err != nil { return err @@ -973,7 +973,7 @@ func (dc *DeploymentConfig) expandVariables() error { context := varContext{ groupIndex: iGrp, modIndex: iMod, - dc: *dc, + dc: dc, } err := updateVariables(context, mod.Settings, true) if err != nil { diff --git a/pkg/config/expand_test.go b/pkg/config/expand_test.go index 7030f322d7..97cb524250 100644 --- a/pkg/config/expand_test.go +++ b/pkg/config/expand_test.go @@ -773,7 +773,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { } testVarContext0 := varContext{ - dc: DeploymentConfig{ + dc: &DeploymentConfig{ Config: testBlueprint, }, modIndex: 0, @@ -781,7 +781,7 @@ func (s *MySuite) TestExpandSimpleVariable(c *C) { } testVarContext1 := varContext{ - dc: DeploymentConfig{ + dc: &DeploymentConfig{ Config: testBlueprint, }, modIndex: 0, From e15eeb9db835a316ecca748543d8490fc2720adb Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 18:14:24 -0500 Subject: [PATCH 086/100] Create test_deployment_variable_not_used validator --- pkg/config/config.go | 37 ++++++++++++++++++++++++++++++++++++ pkg/config/config_test.go | 20 +++++++++++++++---- pkg/config/validate.go | 25 ++++++++++++++++++------ pkg/validators/validators.go | 18 ++++++++++++++++++ 4 files changed, 90 insertions(+), 10 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index 35b8221e32..4fd3eb2a8b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -29,6 +29,7 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" ctyJson "github.com/zclconf/go-cty/cty/json" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" "gopkg.in/yaml.v3" @@ -132,6 +133,7 @@ const ( testModuleNotUsedName testZoneInRegionName testApisEnabledName + testDeploymentVariableNotUsedName ) // this enum will be used to control how fatal validator failures will be @@ -176,6 +178,8 @@ func (v validatorName) String() string { return "test_apis_enabled" case testModuleNotUsedName: return "test_module_not_used" + case testDeploymentVariableNotUsedName: + return "test_deployment_variable_not_used" default: return "unknown_validator" } @@ -349,6 +353,39 @@ func (dc *DeploymentConfig) listUnusedModules() map[string][]string { return unusedModules } +func (dc *DeploymentConfig) listUnusedDeploymentVariables() []string { + var usedVars = []string{} + for _, connections := range dc.moduleConnections { + for _, conn := range connections { + if conn.kind == deploymentConnection { + for _, v := range conn.sharedVariables { + if !slices.Contains(usedVars, v) { + usedVars = append(usedVars, v) + } + + } + } + } + } + + // variables that are either required or automatically constructed and + // applied; these should not be returned as unused otherwise no blueprints + // could be validated + requiredVars := []string{"labels", "deployment_name"} + var unusedVars = []string{} + for _, v := range maps.Keys(dc.Config.Vars) { + if slices.Contains(requiredVars, v) { + continue + } + found := slices.Contains(usedVars, v) + if !found { + unusedVars = append(unusedVars, v) + } + } + + return unusedVars +} + func (dc *DeploymentConfig) checkMovedModules() error { var err error for _, grp := range dc.Config.DeploymentGroups { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index f2612a4664..8949d6122e 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -338,6 +338,9 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { moduleConnections: make(map[string][]ModConnection), } + dc.addSettingsToModules() + dc.addMetadataToModules() + dc.addDefaultValidators() reader := modulereader.Factory("terraform") reader.SetInfo(testModuleSource0, testModuleInfo0) reader.SetInfo(testModuleSource1, testModuleInfo1) @@ -456,6 +459,19 @@ func (s *MySuite) TestListUnusedModules(c *C) { c.Assert(got["usingModule"], HasLen, 2) } +func (s *MySuite) TestListUnusedDeploymentVariables(c *C) { + dc := getDeploymentConfigForTest() + dc.applyGlobalVariables() + dc.expandVariables() + unusedVars := dc.listUnusedDeploymentVariables() + c.Assert(unusedVars, DeepEquals, []string{"project_id"}) + dc = getMultiGroupDeploymentConfig() + dc.applyGlobalVariables() + dc.expandVariables() + unusedVars = dc.listUnusedDeploymentVariables() + c.Assert(unusedVars, DeepEquals, []string{"unused_key"}) +} + func (s *MySuite) TestAddKindToModules(c *C) { /* Test addKindToModules() works when nothing to do */ dc := getBasicDeploymentConfigWithTestModule() @@ -503,10 +519,6 @@ func (s *MySuite) TestModuleConnections(c *C) { modID0 := dc.Config.DeploymentGroups[0].Modules[0].ID modID1 := dc.Config.DeploymentGroups[1].Modules[0].ID - dc.addSettingsToModules() - dc.addMetadataToModules() - dc.addDefaultValidators() - err := dc.applyUseModules() c.Assert(err, IsNil) err = dc.applyGlobalVariables() diff --git a/pkg/config/validate.go b/pkg/config/validate.go index e09e4b7457..6ca6388944 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -284,12 +284,13 @@ func (dc DeploymentConfig) validateModuleSettings() error { func (dc *DeploymentConfig) getValidators() map[string]func(validatorConfig) error { allValidators := map[string]func(validatorConfig) error{ - testApisEnabledName.String(): dc.testApisEnabled, - testProjectExistsName.String(): dc.testProjectExists, - testRegionExistsName.String(): dc.testRegionExists, - testZoneExistsName.String(): dc.testZoneExists, - testZoneInRegionName.String(): dc.testZoneInRegion, - testModuleNotUsedName.String(): dc.testModuleNotUsed, + testApisEnabledName.String(): dc.testApisEnabled, + testProjectExistsName.String(): dc.testProjectExists, + testRegionExistsName.String(): dc.testRegionExists, + testZoneExistsName.String(): dc.testZoneExists, + testZoneInRegionName.String(): dc.testZoneInRegion, + testModuleNotUsedName.String(): dc.testModuleNotUsed, + testDeploymentVariableNotUsedName.String(): dc.testDeploymentVariableNotUsed, } return allValidators } @@ -475,6 +476,18 @@ func (dc *DeploymentConfig) testModuleNotUsed(c validatorConfig) error { return nil } +func (dc *DeploymentConfig) testDeploymentVariableNotUsed(c validatorConfig) error { + if err := c.check(testDeploymentVariableNotUsedName, []string{}); err != nil { + return err + } + + if err := validators.TestDeploymentVariablesNotUsed(dc.listUnusedDeploymentVariables()); err != nil { + log.Print(err) + return fmt.Errorf(funcErrorMsgTemplate, testDeploymentVariableNotUsedName.String()) + } + return nil +} + // return the actual value of a global variable specified by the literal // variable inputReference in form ((var.project_id)) // if it is a literal global variable defined as a string, return value as string diff --git a/pkg/validators/validators.go b/pkg/validators/validators.go index 1829095597..12d3a6fd9a 100644 --- a/pkg/validators/validators.go +++ b/pkg/validators/validators.go @@ -39,6 +39,8 @@ const computeDisabledMsg = "the Compute Engine API must be enabled in project %s const serviceDisabledMsg = "the Service Usage API must be enabled in project %s to validate that all APIs needed by the blueprint are enabled" const unusedModuleMsg = "module %s uses module %s, but matching setting and outputs were not found. This may be because the value is set explicitly or set by a prior used module" const unusedModuleError = "One or more used modules could not have their settings and outputs linked." +const unusedDeploymentVariableMsg = "the deployment variable \"%s\" was not used in this blueprint" +const unusedDeploymentVariableError = "one or more deployment variables was not used by any modules" func handleClientError(e error) error { if strings.Contains(e.Error(), "could not find default credentials") { @@ -49,6 +51,22 @@ func handleClientError(e error) error { return e } +// TestDeploymentVariablesNotUsed errors if there are any unused deployment +// variables and prints any to the output for the user +func TestDeploymentVariablesNotUsed(unusedVariables []string) error { + var foundUnused bool + for _, v := range unusedVariables { + foundUnused = true + log.Printf(unusedDeploymentVariableMsg, v) + } + + if foundUnused { + return fmt.Errorf(unusedDeploymentVariableError) + } + + return nil +} + // TestModuleNotUsed validates that all modules referenced in the "use" field // of the blueprint are actually used, i.e. the outputs and settings are // connected. From 717bad0bda0af0f5f1ae77e6c943f96b7e3f308a Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 18:14:25 -0500 Subject: [PATCH 087/100] Document new test_deployment_variable_not_used validator --- docs/blueprint-validation.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/blueprint-validation.md b/docs/blueprint-validation.md index 9faa3463e7..f7dc6be063 100644 --- a/docs/blueprint-validation.md +++ b/docs/blueprint-validation.md @@ -12,10 +12,10 @@ project(s): One can [explicitly define validators](#explicit-validators), however, the expectation is that the implicit behavior will be useful for most users. When implicit, a validator is added if all deployment variables matching its inputs -is defined. The `test_apis_enabled` validator is always enabled because it reads -the entire blueprint and does not require any specific deployment variable. If -`project_id`, `region`, and `zone` are defined as deployment variables, then the -following validators are enabled: +is defined. Validators listed below that have no inputs are always enabled +because they read the entire blueprint and do not require any specific +deployment variable. If `project_id`, `region`, and `zone` are defined as +deployment variables, then the following validators are enabled: ```yaml validators: @@ -76,7 +76,16 @@ Each validator is described below: * FAIL: if either region or zone do not exist or the zone is not within the region * Common failure: changing 1 value but not the other - * Manual test: `gcloud compute regions describe us-central1 --format="text(zones)" --project $(vars.project_id) + * Manual test: `gcloud compute regions describe us-central1 --format="text(zones)" --project $(vars.project_id)` +* `test_module_not_used` + * Inputs: none; reads whole blueprint + * PASS: if all instances of use keyword pass matching variables + * FAIL: if any instances of use keyword do not pass matching variables +* `test_deployment_variable_not_used` + * Inputs: none; reads whole blueprint + * PASS: if all deployment variables are automatically or explicitly used in + blueprint + * FAIL: if any deployment variable is unused in the blueprint ### Explicit validators From 30f5bd8f0696a35ec5561bac5fa2669eec4799b4 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 18:14:25 -0500 Subject: [PATCH 088/100] Add test_deployment_variable_not_used to default validators --- pkg/config/expand.go | 4 ++++ pkg/config/validator_test.go | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 2240985804..7a9b31545d 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -1012,6 +1012,10 @@ func (dc *DeploymentConfig) addDefaultValidators() error { Validator: testModuleNotUsedName.String(), Inputs: map[string]interface{}{}, }) + defaults = append(defaults, validatorConfig{ + Validator: testDeploymentVariableNotUsedName.String(), + Inputs: map[string]interface{}{}, + }) // always add the project ID validator before subsequent validators that can // only succeed if credentials can access the project. If the project ID diff --git a/pkg/config/validator_test.go b/pkg/config/validator_test.go index 73714574ef..9fadeffc61 100644 --- a/pkg/config/validator_test.go +++ b/pkg/config/validator_test.go @@ -183,17 +183,17 @@ func (s *MySuite) TestValidateOutputs(c *C) { func (s *MySuite) TestAddDefaultValidators(c *C) { dc := getDeploymentConfigForTest() dc.addDefaultValidators() - c.Assert(dc.Config.Validators, HasLen, 3) + c.Assert(dc.Config.Validators, HasLen, 4) dc.Config.Validators = nil dc.Config.Vars["region"] = "us-central1" dc.addDefaultValidators() - c.Assert(dc.Config.Validators, HasLen, 4) + c.Assert(dc.Config.Validators, HasLen, 5) dc.Config.Validators = nil dc.Config.Vars["zone"] = "us-central1-c" dc.addDefaultValidators() - c.Assert(dc.Config.Validators, HasLen, 6) + c.Assert(dc.Config.Validators, HasLen, 7) } // return the actual value of a global variable specified by the literal From 09987b2d72148e0e15d65959d8502db6e68e1bf4 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Fri, 31 Mar 2023 18:14:25 -0500 Subject: [PATCH 089/100] Address test_deployment_variable_not_used failures in test blueprints --- tools/validate_configs/test_configs/apt-collision.yaml | 1 - tools/validate_configs/test_configs/complex-data.yaml | 5 +++++ tools/validate_configs/test_configs/dashboards.yaml | 2 -- tools/validate_configs/test_configs/hpc-cluster-project.yaml | 1 - tools/validate_configs/test_configs/new_project.yaml | 4 ++-- tools/validate_configs/test_configs/pbs-unwrapped.yaml | 1 - 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/validate_configs/test_configs/apt-collision.yaml b/tools/validate_configs/test_configs/apt-collision.yaml index 1cfd04c60c..3b4e2a7300 100644 --- a/tools/validate_configs/test_configs/apt-collision.yaml +++ b/tools/validate_configs/test_configs/apt-collision.yaml @@ -23,7 +23,6 @@ vars: zone: us-central1-a name_prefix: workstation machine_type: a2-highgpu-2g - gpu_per_vm: 2 instance_image: family: pytorch-1-10-gpu-debian-10 project: ml-images diff --git a/tools/validate_configs/test_configs/complex-data.yaml b/tools/validate_configs/test_configs/complex-data.yaml index 5863c6080e..7c2f9ae9c4 100644 --- a/tools/validate_configs/test_configs/complex-data.yaml +++ b/tools/validate_configs/test_configs/complex-data.yaml @@ -16,6 +16,11 @@ blueprint_name: complex_data +validators: +- validator: test_deployment_variable_not_used + inputs: {} + skip: true + vars: project_id: ## Set GCP Project ID Here ## deployment_name: test_complex-data diff --git a/tools/validate_configs/test_configs/dashboards.yaml b/tools/validate_configs/test_configs/dashboards.yaml index 1e4a8b80f2..aa589ef376 100644 --- a/tools/validate_configs/test_configs/dashboards.yaml +++ b/tools/validate_configs/test_configs/dashboards.yaml @@ -19,8 +19,6 @@ blueprint_name: dashboards vars: project_id: ## Set GCP Project ID Here ## deployment_name: dashboards-test - region: europe-west4 - zone: europe-west4-a deployment_groups: - group: primary diff --git a/tools/validate_configs/test_configs/hpc-cluster-project.yaml b/tools/validate_configs/test_configs/hpc-cluster-project.yaml index 2feb7ddff5..b900649f2a 100644 --- a/tools/validate_configs/test_configs/hpc-cluster-project.yaml +++ b/tools/validate_configs/test_configs/hpc-cluster-project.yaml @@ -21,7 +21,6 @@ vars: deployment_name: hpc-slurm-project region: europe-west4 zone: europe-west4-a - slurm_sa: slurm-sa terraform_backend_defaults: type: gcs diff --git a/tools/validate_configs/test_configs/new_project.yaml b/tools/validate_configs/test_configs/new_project.yaml index c62b9d3984..069ac9587d 100644 --- a/tools/validate_configs/test_configs/new_project.yaml +++ b/tools/validate_configs/test_configs/new_project.yaml @@ -17,6 +17,7 @@ blueprint_name: new_project vars: + project_id: test_project deployment_name: new_project_deployment deployment_groups: @@ -25,7 +26,6 @@ deployment_groups: - id: project source: ./community/modules/project/new-project settings: - project_id: test_project folder_id: 334688113020 # random number - billing_account: "111110-M2N704-854685" # random billing number + billing_account: 111110-M2N704-854685 # random billing number org_id: 123456789 # random org id diff --git a/tools/validate_configs/test_configs/pbs-unwrapped.yaml b/tools/validate_configs/test_configs/pbs-unwrapped.yaml index 141d7774a2..2809345870 100644 --- a/tools/validate_configs/test_configs/pbs-unwrapped.yaml +++ b/tools/validate_configs/test_configs/pbs-unwrapped.yaml @@ -25,7 +25,6 @@ vars: execution_host_count: 10 execution_hostname_prefix: pbs-execution pbs_client_rpm_url: gs://replace-pbspro-rpm-bucket/pbspro-client-2021.1.3.20220217134230-0.el7.x86_64.rpm - pbs_devel_rpm_url: gs://replace-pbspro-rpm-bucket/pbspro-devel-2021.1.3.20220217134230-0.el7.x86_64.rpm pbs_execution_rpm_url: gs://replace-pbspro-rpm-bucket/pbspro-execution-2021.1.3.20220217134230-0.el7.x86_64.rpm pbs_server_rpm_url: gs://replace-pbspro-rpm-bucket/pbspro-server-2021.1.3.20220217134230-0.el7.x86_64.rpm From 46424f5cf03f193781db5b1340fbeee726941740 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Apr 2023 11:00:32 +0000 Subject: [PATCH 090/100] Bump google-auth from 2.17.0 to 2.17.1 in /community/front-end/ofe Bumps [google-auth](https://github.com/googleapis/google-auth-library-python) from 2.17.0 to 2.17.1. - [Release notes](https://github.com/googleapis/google-auth-library-python/releases) - [Changelog](https://github.com/googleapis/google-auth-library-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/googleapis/google-auth-library-python/compare/v2.17.0...v2.17.1) --- updated-dependencies: - dependency-name: google-auth dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 1cc9cdbcce..9b1b99202c 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -23,7 +23,7 @@ djangorestframework==3.14.0 filelock==3.10.7 google-api-core==2.11.0 google-api-python-client==2.83.0 -google-auth==2.17.0 +google-auth==2.17.1 google-auth-httplib2==0.1.0 google-cloud-billing==1.10.1 google-cloud-core==2.3.2 From fe27ddfb380b484e7cb1c0aefbb3ca8180cb9077 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Apr 2023 11:01:12 +0000 Subject: [PATCH 091/100] Bump django-allauth from 0.53.1 to 0.54.0 in /community/front-end/ofe Bumps [django-allauth](https://github.com/pennersr/django-allauth) from 0.53.1 to 0.54.0. - [Release notes](https://github.com/pennersr/django-allauth/releases) - [Changelog](https://github.com/pennersr/django-allauth/blob/master/ChangeLog.rst) - [Commits](https://github.com/pennersr/django-allauth/compare/0.53.1...0.54.0) --- updated-dependencies: - dependency-name: django-allauth dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index 1cc9cdbcce..b12807508e 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -17,7 +17,7 @@ distlib==0.3.6 # django-revproxy==0.11.0 released but not yet in pypi git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 Django==4.1.7 -django-allauth==0.53.1 +django-allauth==0.54.0 django-extensions==3.1.5 djangorestframework==3.14.0 filelock==3.10.7 From 0872ea4a0ef13d88c04a18ff9397e9855003532a Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Sat, 1 Apr 2023 22:42:03 -0500 Subject: [PATCH 092/100] Address feedback from #1101 --- docs/blueprint-validation.md | 58 +++++++++++++++++++----------------- pkg/config/config.go | 30 ++++++++----------- pkg/validators/validators.go | 4 +-- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/docs/blueprint-validation.md b/docs/blueprint-validation.md index f7dc6be063..1687b62e58 100644 --- a/docs/blueprint-validation.md +++ b/docs/blueprint-validation.md @@ -12,32 +12,8 @@ project(s): One can [explicitly define validators](#explicit-validators), however, the expectation is that the implicit behavior will be useful for most users. When implicit, a validator is added if all deployment variables matching its inputs -is defined. Validators listed below that have no inputs are always enabled -because they read the entire blueprint and do not require any specific -deployment variable. If `project_id`, `region`, and `zone` are defined as -deployment variables, then the following validators are enabled: - -```yaml -validators: -- validator: test_project_exists - inputs: - project_id: $(vars.project_id) -- validator: test_apis_enabled - inputs: {} -- validator: test_region_exists - inputs: - project_id: $(vars.project_id) - region: $(vars.region) -- validator: test_zone_exists - inputs: - project_id: $(vars.project_id) - zone: $(vars.zone) -- validator: test_zone_in_region - inputs: - project_id: $(vars.project_id) - zone: $(vars.zone) - region: $(vars.region) -``` +are defined. Validators that have no inputs are always enabled by default +because they do not require any specific deployment variable. Each validator is described below: @@ -90,7 +66,35 @@ Each validator is described below: ### Explicit validators Validators can be overwritten and supplied with alternative input values, -however they are limited to the set of functions defined above. +however they are limited to the set of functions defined above. As an example, +the default validators added when `project_id`, `region`, and `zone` are defined +is: + +```yaml +validators: + - validator: test_module_not_used + inputs: {} + - validator: test_deployment_variable_not_used + inputs: {} + - validator: test_project_exists + inputs: + project_id: $(vars.project_id) + - validator: test_apis_enabled + inputs: {} + - validator: test_region_exists + inputs: + project_id: $(vars.project_id) + region: $(vars.region) + - validator: test_zone_exists + inputs: + project_id: $(vars.project_id) + zone: $(vars.zone) + - validator: test_zone_in_region + inputs: + project_id: $(vars.project_id) + region: $(vars.region) + zone: $(vars.zone) +``` ### Skipping or disabling validators diff --git a/pkg/config/config.go b/pkg/config/config.go index 4fd3eb2a8b..e5f501ec89 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -29,7 +29,6 @@ import ( "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/gocty" ctyJson "github.com/zclconf/go-cty/cty/json" - "golang.org/x/exp/maps" "golang.org/x/exp/slices" "gopkg.in/yaml.v3" @@ -354,32 +353,27 @@ func (dc *DeploymentConfig) listUnusedModules() map[string][]string { } func (dc *DeploymentConfig) listUnusedDeploymentVariables() []string { - var usedVars = []string{} + // these variables are required or automatically constructed and applied; + // these should not be listed unused otherwise no blueprints are valid + var usedVars = map[string]bool{ + "labels": true, + "deployment_name": true, + } + for _, connections := range dc.moduleConnections { for _, conn := range connections { if conn.kind == deploymentConnection { for _, v := range conn.sharedVariables { - if !slices.Contains(usedVars, v) { - usedVars = append(usedVars, v) - } - + usedVars[v] = true } } } } - // variables that are either required or automatically constructed and - // applied; these should not be returned as unused otherwise no blueprints - // could be validated - requiredVars := []string{"labels", "deployment_name"} - var unusedVars = []string{} - for _, v := range maps.Keys(dc.Config.Vars) { - if slices.Contains(requiredVars, v) { - continue - } - found := slices.Contains(usedVars, v) - if !found { - unusedVars = append(unusedVars, v) + unusedVars := []string{} + for k := range dc.Config.Vars { + if _, ok := usedVars[k]; !ok { + unusedVars = append(unusedVars, k) } } diff --git a/pkg/validators/validators.go b/pkg/validators/validators.go index 12d3a6fd9a..307a9d8fcd 100644 --- a/pkg/validators/validators.go +++ b/pkg/validators/validators.go @@ -54,13 +54,11 @@ func handleClientError(e error) error { // TestDeploymentVariablesNotUsed errors if there are any unused deployment // variables and prints any to the output for the user func TestDeploymentVariablesNotUsed(unusedVariables []string) error { - var foundUnused bool for _, v := range unusedVariables { - foundUnused = true log.Printf(unusedDeploymentVariableMsg, v) } - if foundUnused { + if len(unusedVariables) > 0 { return fmt.Errorf(unusedDeploymentVariableError) } From 4b876e74ba8346c9701a185b6537f594472686fc Mon Sep 17 00:00:00 2001 From: Thiago Sgobe Date: Mon, 3 Apr 2023 11:01:05 -0300 Subject: [PATCH 093/100] add disk size --- community/modules/compute/htcondor-execute-point/README.md | 1 + community/modules/compute/htcondor-execute-point/main.tf | 1 + .../modules/compute/htcondor-execute-point/variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/community/modules/compute/htcondor-execute-point/README.md b/community/modules/compute/htcondor-execute-point/README.md index b660a3247d..52f0151922 100644 --- a/community/modules/compute/htcondor-execute-point/README.md +++ b/community/modules/compute/htcondor-execute-point/README.md @@ -180,6 +180,7 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [deployment\_name](#input\_deployment\_name) | HPC Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | | [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | | [image](#input\_image) | HTCondor execute point VM image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | | [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | diff --git a/community/modules/compute/htcondor-execute-point/main.tf b/community/modules/compute/htcondor-execute-point/main.tf index db99f23065..23dfe639eb 100644 --- a/community/modules/compute/htcondor-execute-point/main.tf +++ b/community/modules/compute/htcondor-execute-point/main.tf @@ -55,6 +55,7 @@ module "execute_point_instance_template" { labels = var.labels machine_type = var.machine_type + disk_size_gb = var.disk_size_gb preemptible = var.spot startup_script = var.startup_script metadata = local.metadata diff --git a/community/modules/compute/htcondor-execute-point/variables.tf b/community/modules/compute/htcondor-execute-point/variables.tf index 93037c5a86..29468aaa61 100644 --- a/community/modules/compute/htcondor-execute-point/variables.tf +++ b/community/modules/compute/htcondor-execute-point/variables.tf @@ -146,3 +146,9 @@ variable "spot" { type = bool default = false } + +variable "disk_size_gb" { + description = "Boot disk size in GB" + type = string + default = "100" +} From 0901b19336335cc65896ed2a5ce2e81b7f415c56 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 3 Apr 2023 09:48:18 -0500 Subject: [PATCH 094/100] Simplify test_module_not_used implementation --- pkg/validators/validators.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/validators/validators.go b/pkg/validators/validators.go index 307a9d8fcd..3fd9cb0505 100644 --- a/pkg/validators/validators.go +++ b/pkg/validators/validators.go @@ -69,15 +69,13 @@ func TestDeploymentVariablesNotUsed(unusedVariables []string) error { // of the blueprint are actually used, i.e. the outputs and settings are // connected. func TestModuleNotUsed(unusedModules map[string][]string) error { - foundUnused := false for mod, unusedMods := range unusedModules { - foundUnused = true for _, unusedMod := range unusedMods { log.Printf(unusedModuleMsg, mod, unusedMod) } } - if foundUnused { + if len(unusedModules) > 0 { return fmt.Errorf(unusedModuleError) } From 2a6bc7677939ac0dc9f7076f65a3df3a8e435dee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:09:29 +0000 Subject: [PATCH 095/100] Bump django-extensions from 3.1.5 to 3.2.1 in /community/front-end/ofe Bumps [django-extensions](https://github.com/django-extensions/django-extensions) from 3.1.5 to 3.2.1. - [Release notes](https://github.com/django-extensions/django-extensions/releases) - [Changelog](https://github.com/django-extensions/django-extensions/blob/main/CHANGELOG.md) - [Commits](https://github.com/django-extensions/django-extensions/compare/3.1.5...3.2.1) --- updated-dependencies: - dependency-name: django-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- community/front-end/ofe/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/community/front-end/ofe/requirements.txt b/community/front-end/ofe/requirements.txt index b12807508e..c4234f6da3 100644 --- a/community/front-end/ofe/requirements.txt +++ b/community/front-end/ofe/requirements.txt @@ -18,7 +18,7 @@ distlib==0.3.6 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787 Django==4.1.7 django-allauth==0.54.0 -django-extensions==3.1.5 +django-extensions==3.2.1 djangorestframework==3.14.0 filelock==3.10.7 google-api-core==2.11.0 From 095767284052b652375b467c2c9bbc0009a68c23 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 3 Apr 2023 12:23:48 -0500 Subject: [PATCH 096/100] Track explicit connections between modules --- pkg/config/config.go | 15 ++++-- pkg/config/config_test.go | 100 ++++++++++++++++++++++++++++++++------ pkg/config/expand.go | 12 +++++ 3 files changed, 109 insertions(+), 18 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index e5f501ec89..f1aa891167 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -263,19 +263,26 @@ type Blueprint struct { TerraformBackendDefaults TerraformBackend `yaml:"terraform_backend_defaults"` } -// connectionKind defines the kind of module connection, defined by the source -// of the connection. Currently, only Use is supported. +// connectionKind defines tracks graph edges between modules and from modules to +// deployment variables: +// +// use: created via module-module use keyword +// deployment: created by a module setting equal to $(vars.name) +// explicit: created by a module setting equal to $(mod_id.output) +// +// no attempt is made to track edges made via Toolkit literal strings presently +// required when wanting to index a list or map "((mod_id.output[0]))" type connectionKind int const ( undefinedConnection connectionKind = iota useConnection deploymentConnection - // explicitConnection + explicitConnection ) func (c connectionKind) IsValid() bool { - return c == useConnection || c == deploymentConnection + return c == useConnection || c == deploymentConnection || c == explicitConnection } // ModConnection defines details about connections between modules. Currently, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 8949d6122e..f7ca5a48c2 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -255,8 +255,14 @@ func getBasicDeploymentConfigWithTestModule() DeploymentConfig { func getMultiGroupDeploymentConfig() DeploymentConfig { testModuleSource0 := filepath.Join(tmpTestDir, "module0") testModuleSource1 := filepath.Join(tmpTestDir, "module1") + testModuleSource2 := filepath.Join(tmpTestDir, "module2") - matchingName := "test_match" + matchingIntergroupName := "test_inter_0" + matchingIntragroupName0 := "test_intra_0" + matchingIntragroupName1 := "test_intra_1" + matchingIntragroupName2 := "test_intra_2" + + altProjectIDSetting := "host_project_id" testModuleInfo0 := modulereader.ModuleInfo{ Inputs: []modulereader.VarInfo{ @@ -265,40 +271,78 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { Type: "string", }, { - Name: "host_project_id", + Name: altProjectIDSetting, Type: "string", }, }, Outputs: []modulereader.VarInfo{ { - Name: matchingName, + Name: matchingIntergroupName, + }, + { + Name: matchingIntragroupName0, + }, + { + Name: matchingIntragroupName1, + }, + { + Name: matchingIntragroupName2, }, }, } testModuleInfo1 := modulereader.ModuleInfo{ + Inputs: []modulereader.VarInfo{ + { + Name: matchingIntragroupName0, + }, + { + Name: matchingIntragroupName1, + }, + { + Name: matchingIntragroupName2, + }, + }, + Outputs: []modulereader.VarInfo{}, + } + + testModuleInfo2 := modulereader.ModuleInfo{ Inputs: []modulereader.VarInfo{ { Name: "deployment_name", Type: "string", }, { - Name: matchingName, + Name: matchingIntergroupName, }, }, Outputs: []modulereader.VarInfo{}, } + dg0Name := "primary" + modID0 := "TestModule0" testDeploymentGroup0 := DeploymentGroup{ - Name: "primary", + Name: dg0Name, Modules: []Module{ { - ID: "TestModule0", + ID: modID0, Kind: "terraform", Source: testModuleSource0, Settings: map[string]interface{}{ - "host_project_id": "$(vars.project_id)", + altProjectIDSetting: "$(vars.project_id)", + }, + Outputs: []string{matchingIntergroupName}, + }, + { + ID: "TestModule1", + Kind: "terraform", + Source: testModuleSource1, + Settings: map[string]interface{}{ + matchingIntragroupName1: "explicit-intra-value", + matchingIntragroupName2: fmt.Sprintf("$(%s.%s)", modID0, matchingIntragroupName2), + }, + Use: []string{ + fmt.Sprintf("%s.%s", dg0Name, modID0), }, - Outputs: []string{matchingName}, }, }, } @@ -306,9 +350,9 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { Name: "secondary", Modules: []Module{ { - ID: "TestModule1", + ID: "TestModule2", Kind: "terraform", - Source: testModuleSource1, + Source: testModuleSource2, Settings: map[string]interface{}{}, Use: []string{ fmt.Sprintf("%s.%s", testDeploymentGroup0.Name, testDeploymentGroup0.Modules[0].ID), @@ -330,9 +374,10 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { ModulesInfo: map[string]map[string]modulereader.ModuleInfo{ testDeploymentGroup0.Name: { testModuleSource0: testModuleInfo0, + testModuleSource1: testModuleInfo1, }, testDeploymentGroup1.Name: { - testModuleSource1: testModuleInfo1, + testModuleSource2: testModuleInfo2, }, }, moduleConnections: make(map[string][]ModConnection), @@ -344,6 +389,7 @@ func getMultiGroupDeploymentConfig() DeploymentConfig { reader := modulereader.Factory("terraform") reader.SetInfo(testModuleSource0, testModuleInfo0) reader.SetInfo(testModuleSource1, testModuleInfo1) + reader.SetInfo(testModuleSource2, testModuleInfo2) return dc } @@ -517,7 +563,8 @@ func (s *MySuite) TestAddKindToModules(c *C) { func (s *MySuite) TestModuleConnections(c *C) { dc := getMultiGroupDeploymentConfig() modID0 := dc.Config.DeploymentGroups[0].Modules[0].ID - modID1 := dc.Config.DeploymentGroups[1].Modules[0].ID + modID1 := dc.Config.DeploymentGroups[0].Modules[1].ID + modID2 := dc.Config.DeploymentGroups[1].Modules[0].ID err := dc.applyUseModules() c.Assert(err, IsNil) @@ -561,17 +608,42 @@ func (s *MySuite) TestModuleConnections(c *C) { toModuleID: "TestModule0", fromModuleID: "TestModule1", toGroupID: "primary", + fromGroupID: "primary", + explicit: true, + }, + kind: useConnection, + sharedVariables: []string{"test_intra_0"}, + }, + { + ref: varReference{ + name: "test_intra_2", + toModuleID: "TestModule0", + fromModuleID: "TestModule1", + toGroupID: "primary", + fromGroupID: "primary", + explicit: false, + }, + kind: explicitConnection, + sharedVariables: []string{"test_intra_2"}, + }, + }, + modID2: { + { + ref: modReference{ + toModuleID: "TestModule0", + fromModuleID: "TestModule2", + toGroupID: "primary", fromGroupID: "secondary", explicit: true, }, kind: useConnection, - sharedVariables: []string{"test_match"}, + sharedVariables: []string{"test_inter_0"}, }, { ref: varReference{ name: "deployment_name", toModuleID: "vars", - fromModuleID: "TestModule1", + fromModuleID: "TestModule2", toGroupID: "deployment", fromGroupID: "secondary", explicit: false, diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 7a9b31545d..034a180789 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -829,6 +829,18 @@ func expandSimpleVariable(context varContext, trackModuleGraph bool) (string, er case varRef.fromGroupID: // intragroup reference can make direct reference to module output expandedVariable = fmt.Sprintf("((module.%s.%s))", varRef.toModuleID, varRef.name) + if trackModuleGraph { + var found bool + for _, conn := range context.dc.moduleConnections[varRef.fromModuleID] { + if slices.Contains(conn.sharedVariables, varRef.name) { + found = true + break + } + } + if !found { + context.dc.addModuleConnection(varRef, explicitConnection, []string{varRef.name}) + } + } default: // intergroup reference; begin by finding the target module in blueprint From eed60a0b13b7e2553a6110863bf74dedc7ae687e Mon Sep 17 00:00:00 2001 From: Thiago Sgobe Date: Mon, 3 Apr 2023 15:53:02 -0300 Subject: [PATCH 097/100] Change disk_size_gb var type --- community/modules/compute/htcondor-execute-point/README.md | 2 +- community/modules/compute/htcondor-execute-point/variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/community/modules/compute/htcondor-execute-point/README.md b/community/modules/compute/htcondor-execute-point/README.md index 52f0151922..d83c0d2c86 100644 --- a/community/modules/compute/htcondor-execute-point/README.md +++ b/community/modules/compute/htcondor-execute-point/README.md @@ -180,7 +180,7 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [deployment\_name](#input\_deployment\_name) | HPC Toolkit deployment name. HTCondor cloud resource names will include this value. | `string` | n/a | yes | -| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `string` | `"100"` | no | +| [disk\_size\_gb](#input\_disk\_size\_gb) | Boot disk size in GB | `number` | `100` | no | | [enable\_oslogin](#input\_enable\_oslogin) | Enable or Disable OS Login with "ENABLE" or "DISABLE". Set to "INHERIT" to inherit project OS Login setting. | `string` | `"ENABLE"` | no | | [image](#input\_image) | HTCondor execute point VM image | 
object({
    family  = string,
    project = string
  })
| 
{
  "family": "hpc-centos-7",
  "project": "cloud-hpc-image-public"
}
| no | | [labels](#input\_labels) | Labels to add to HTConodr execute points | `map(string)` | n/a | yes | diff --git a/community/modules/compute/htcondor-execute-point/variables.tf b/community/modules/compute/htcondor-execute-point/variables.tf index 29468aaa61..683c89f4d6 100644 --- a/community/modules/compute/htcondor-execute-point/variables.tf +++ b/community/modules/compute/htcondor-execute-point/variables.tf @@ -149,6 +149,6 @@ variable "spot" { variable "disk_size_gb" { description = "Boot disk size in GB" - type = string - default = "100" + type = number + default = 100 } From 874ac5ed86d00b8906e01d51644f40c9b0669386 Mon Sep 17 00:00:00 2001 From: Tom Downes Date: Mon, 3 Apr 2023 13:57:00 -0500 Subject: [PATCH 098/100] Address feedback from #1109 --- pkg/config/expand.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/config/expand.go b/pkg/config/expand.go index 034a180789..da7ba960df 100644 --- a/pkg/config/expand.go +++ b/pkg/config/expand.go @@ -832,6 +832,9 @@ func expandSimpleVariable(context varContext, trackModuleGraph bool) (string, er if trackModuleGraph { var found bool for _, conn := range context.dc.moduleConnections[varRef.fromModuleID] { + if conn.kind != useConnection { + continue + } if slices.Contains(conn.sharedVariables, varRef.name) { found = true break From 5f3475af8b2770be415e12b95eb6f75f0904d19e Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Mon, 3 Apr 2023 13:42:12 -0700 Subject: [PATCH 099/100] Test infra automation. Init commit (#1105) Add terraform module `tools/cloud-build/provision` to manage all test infrastructure. In this PR define following triggers and schedule them: * `DAILY-project-cleanup` * `WEEKLY-build-dependency-check` * `WEEKLY-builder-image` * `ZEBUG-fast-build-failure` * `ZEBUG-fast-build-success` --- tools/cloud-build/README.md | 1 + tools/cloud-build/provision/README.md | 48 ++++++++++++++++++ tools/cloud-build/provision/daily-cleanup.tf | 38 ++++++++++++++ tools/cloud-build/provision/main.tf | 25 ++++++++++ tools/cloud-build/provision/providers.tf | 27 ++++++++++ .../provision/trigger-schedule/README.md | 37 ++++++++++++++ .../provision/trigger-schedule/main.tf | 36 ++++++++++++++ .../provision/trigger-schedule/variables.tf | 27 ++++++++++ .../provision/trigger-schedule/versions.tf | 24 +++++++++ tools/cloud-build/provision/variables.tf | 38 ++++++++++++++ tools/cloud-build/provision/versions.tf | 28 +++++++++++ .../weekly_build_dependency_check.tf | 38 ++++++++++++++ .../provision/weekly_builder_image.tf | 38 ++++++++++++++ tools/cloud-build/provision/zebug.tf | 49 +++++++++++++++++++ 14 files changed, 454 insertions(+) create mode 100644 tools/cloud-build/provision/README.md create mode 100644 tools/cloud-build/provision/daily-cleanup.tf create mode 100644 tools/cloud-build/provision/main.tf create mode 100644 tools/cloud-build/provision/providers.tf create mode 100644 tools/cloud-build/provision/trigger-schedule/README.md create mode 100644 tools/cloud-build/provision/trigger-schedule/main.tf create mode 100644 tools/cloud-build/provision/trigger-schedule/variables.tf create mode 100644 tools/cloud-build/provision/trigger-schedule/versions.tf create mode 100644 tools/cloud-build/provision/variables.tf create mode 100644 tools/cloud-build/provision/versions.tf create mode 100644 tools/cloud-build/provision/weekly_build_dependency_check.tf create mode 100644 tools/cloud-build/provision/weekly_builder_image.tf create mode 100644 tools/cloud-build/provision/zebug.tf diff --git a/tools/cloud-build/README.md b/tools/cloud-build/README.md index 66fc57a15f..9cae005e3a 100644 --- a/tools/cloud-build/README.md +++ b/tools/cloud-build/README.md @@ -16,3 +16,4 @@ pre-commits on all files. * `project-cleanup.yaml`: Cloud build config that performs a regular cleanup of resources in the test project. +* `provision`: Terraform module that sets up CloudBuild triggers and schedule. diff --git a/tools/cloud-build/provision/README.md b/tools/cloud-build/provision/README.md new file mode 100644 index 0000000000..58ae89d0a4 --- /dev/null +++ b/tools/cloud-build/provision/README.md @@ -0,0 +1,48 @@ +`provision` module creates CloudBuilds triggers and schedules. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | ~> 4.58.0 | +| [google-beta](#requirement\_google-beta) | ~> 4.58.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | ~> 4.58.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [daily\_project\_cleanup\_schedule](#module\_daily\_project\_cleanup\_schedule) | ./trigger-schedule | n/a | +| [weekly\_build\_dependency\_check\_schedule](#module\_weekly\_build\_dependency\_check\_schedule) | ./trigger-schedule | n/a | +| [weekly\_builder\_image\_schedule](#module\_weekly\_builder\_image\_schedule) | ./trigger-schedule | n/a | + +## Resources + +| Name | Type | +|------|------| +| [google_cloudbuild_trigger.daily_project_cleanup](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | +| [google_cloudbuild_trigger.weekly_build_dependency_check](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | +| [google_cloudbuild_trigger.weekly_builder_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | +| [google_cloudbuild_trigger.zebug_fast_build_failure](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | +| [google_cloudbuild_trigger.zebug_fast_build_success](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloudbuild_trigger) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [project\_id](#input\_project\_id) | GCP project ID | `string` | n/a | yes | +| [region](#input\_region) | GCP region | `string` | `"us-central1"` | no | +| [repo\_uri](#input\_repo\_uri) | URI of GitHub repo | `string` | `"https://github.com/GoogleCloudPlatform/hpc-toolkit"` | no | +| [zone](#input\_zone) | GCP zone | `string` | `"us-central1-c"` | no | + +## Outputs + +No outputs. + diff --git a/tools/cloud-build/provision/daily-cleanup.tf b/tools/cloud-build/provision/daily-cleanup.tf new file mode 100644 index 0000000000..23ac40c097 --- /dev/null +++ b/tools/cloud-build/provision/daily-cleanup.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_cloudbuild_trigger" "daily_project_cleanup" { + name = "DAILY-project-cleanup" + description = "A cleanup script to run periodically" + tags = [local.notify_chat_tag] + + git_file_source { + path = "tools/cloud-build/project-cleanup.yaml" + revision = local.ref_develop + uri = var.repo_uri + repo_type = "GITHUB" + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_develop + repo_type = "GITHUB" + } +} + +module "daily_project_cleanup_schedule" { + source = "./trigger-schedule" + trigger = google_cloudbuild_trigger.daily_project_cleanup + schedule = "0 0 * * MON-FRI" +} diff --git a/tools/cloud-build/provision/main.tf b/tools/cloud-build/provision/main.tf new file mode 100644 index 0000000000..c5ffbb50c3 --- /dev/null +++ b/tools/cloud-build/provision/main.tf @@ -0,0 +1,25 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + backend "gcs" { + prefix = "dev-infra-tf/state" + } +} + +locals { + ref_main = "refs/heads/main" + ref_develop = "refs/heads/develop" + notify_chat_tag = "notify_chat_on_failure" +} diff --git a/tools/cloud-build/provision/providers.tf b/tools/cloud-build/provision/providers.tf new file mode 100644 index 0000000000..ec0dc80b57 --- /dev/null +++ b/tools/cloud-build/provision/providers.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google" { + project = var.project_id + zone = var.zone + region = var.region +} + +provider "google-beta" { + project = var.project_id + zone = var.zone + region = var.region +} diff --git a/tools/cloud-build/provision/trigger-schedule/README.md b/tools/cloud-build/provision/trigger-schedule/README.md new file mode 100644 index 0000000000..821fe43c0b --- /dev/null +++ b/tools/cloud-build/provision/trigger-schedule/README.md @@ -0,0 +1,37 @@ +`trigger-schedule` is a helper module to schedule CloudBuild triggers, mimics default behaviour of CloudBuild UI schedule wizzard. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13 | +| [google](#requirement\_google) | ~> 4.58.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | ~> 4.58.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [google_cloud_scheduler_job.schedule](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/cloud_scheduler_job) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [schedule](#input\_schedule) | Describes the schedule on which the job will be executed. | `string` | n/a | yes | +| [trigger](#input\_trigger) | View of google\_cloudbuild\_trigger resource | 
object({
    name    = string
    id      = string
    project = string
  })
| n/a | yes | + +## Outputs + +No outputs. + diff --git a/tools/cloud-build/provision/trigger-schedule/main.tf b/tools/cloud-build/provision/trigger-schedule/main.tf new file mode 100644 index 0000000000..e0610298cd --- /dev/null +++ b/tools/cloud-build/provision/trigger-schedule/main.tf @@ -0,0 +1,36 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_cloud_scheduler_job" "schedule" { + name = "${var.trigger.name}-schedule" + schedule = var.schedule + time_zone = "America/Los_Angeles" + + attempt_deadline = "180s" + retry_config { + max_backoff_duration = "3600s" + max_doublings = 5 + max_retry_duration = "0s" + min_backoff_duration = "5s" + } + + http_target { + http_method = "POST" + uri = "https://cloudbuild.googleapis.com/v1/${var.trigger.id}:run" + headers = { "User-Agent" = "Google-Cloud-Scheduler" } + oauth_token { + service_account_email = "cloud-build-trigger-scheduler@${var.trigger.project}.iam.gserviceaccount.com" + } + } +} diff --git a/tools/cloud-build/provision/trigger-schedule/variables.tf b/tools/cloud-build/provision/trigger-schedule/variables.tf new file mode 100644 index 0000000000..d2ed659a9a --- /dev/null +++ b/tools/cloud-build/provision/trigger-schedule/variables.tf @@ -0,0 +1,27 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "trigger" { + description = "View of google_cloudbuild_trigger resource" + type = object({ + name = string + id = string + project = string + }) +} + +variable "schedule" { + description = "Describes the schedule on which the job will be executed." + type = string +} diff --git a/tools/cloud-build/provision/trigger-schedule/versions.tf b/tools/cloud-build/provision/trigger-schedule/versions.tf new file mode 100644 index 0000000000..c2099900d4 --- /dev/null +++ b/tools/cloud-build/provision/trigger-schedule/versions.tf @@ -0,0 +1,24 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.58.0" + } + } +} diff --git a/tools/cloud-build/provision/variables.tf b/tools/cloud-build/provision/variables.tf new file mode 100644 index 0000000000..5166c7537e --- /dev/null +++ b/tools/cloud-build/provision/variables.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "GCP project ID" + type = string +} + +variable "region" { + description = "GCP region" + type = string + default = "us-central1" +} + +variable "zone" { + description = "GCP zone" + type = string + default = "us-central1-c" +} + +variable "repo_uri" { + description = "URI of GitHub repo" + type = string + default = "https://github.com/GoogleCloudPlatform/hpc-toolkit" +} diff --git a/tools/cloud-build/provision/versions.tf b/tools/cloud-build/provision/versions.tf new file mode 100644 index 0000000000..8f68dab56b --- /dev/null +++ b/tools/cloud-build/provision/versions.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 0.13" + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.58.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.58.0" + } + } +} diff --git a/tools/cloud-build/provision/weekly_build_dependency_check.tf b/tools/cloud-build/provision/weekly_build_dependency_check.tf new file mode 100644 index 0000000000..747ee290f3 --- /dev/null +++ b/tools/cloud-build/provision/weekly_build_dependency_check.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_cloudbuild_trigger" "weekly_build_dependency_check" { + name = "WEEKLY-build-dependency-check" + description = "A set of tests to make sure no extra dependencies creep in" + tags = [local.notify_chat_tag] + + git_file_source { + path = "tools/cloud-build/dependency-checks/hpc-toolkit-go-builder.yaml" + revision = local.ref_develop + uri = var.repo_uri + repo_type = "GITHUB" + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_develop + repo_type = "GITHUB" + } +} + +module "weekly_build_dependency_check_schedule" { + source = "./trigger-schedule" + trigger = google_cloudbuild_trigger.weekly_build_dependency_check + schedule = "0 7 * * MON" +} diff --git a/tools/cloud-build/provision/weekly_builder_image.tf b/tools/cloud-build/provision/weekly_builder_image.tf new file mode 100644 index 0000000000..10d12daf50 --- /dev/null +++ b/tools/cloud-build/provision/weekly_builder_image.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_cloudbuild_trigger" "weekly_builder_image" { + name = "WEEKLY-builder-image" + description = "Builds a container tailored to build and test ghpc" + tags = [local.notify_chat_tag] + + git_file_source { + path = "tools/cloud-build/hpc-toolkit-builder.yaml" + revision = local.ref_develop + uri = var.repo_uri + repo_type = "GITHUB" + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_develop + repo_type = "GITHUB" + } +} + +module "weekly_builder_image_schedule" { + source = "./trigger-schedule" + trigger = google_cloudbuild_trigger.weekly_builder_image + schedule = "0 8 * * MON" +} diff --git a/tools/cloud-build/provision/zebug.tf b/tools/cloud-build/provision/zebug.tf new file mode 100644 index 0000000000..5f5fd31f3a --- /dev/null +++ b/tools/cloud-build/provision/zebug.tf @@ -0,0 +1,49 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "google_cloudbuild_trigger" "zebug_fast_build_failure" { + name = "ZEBUG-fast-build-failure" + description = "A build that always fails fast" + + build { + step { + name = "busybox" + args = ["false"] + } + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_main + repo_type = "GITHUB" + } +} + +resource "google_cloudbuild_trigger" "zebug_fast_build_success" { + name = "ZEBUG-fast-build-success" + description = "A build that always succeeds fast" + + build { + step { + name = "busybox" + args = ["true"] + } + } + + source_to_build { + uri = var.repo_uri + ref = local.ref_main + repo_type = "GITHUB" + } +} From dfcae3aa4beb6717f5e6e78372a6f43ed0e57065 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Fri, 7 Apr 2023 18:09:00 +0000 Subject: [PATCH 100/100] version update --- cmd/root.go | 2 +- .../compute/schedmd-slurm-gcp-v5-node-group/versions.tf | 2 +- .../compute/schedmd-slurm-gcp-v5-partition/versions.tf | 2 +- .../modules/database/slurm-cloudsql-federation/versions.tf | 4 ++-- .../modules/file-system/cloud-storage-bucket/versions.tf | 2 +- community/modules/file-system/nfs-server/versions.tf | 2 +- community/modules/project/service-enablement/versions.tf | 2 +- .../scheduler/SchedMD-slurm-on-gcp-controller/versions.tf | 2 +- .../scheduler/SchedMD-slurm-on-gcp-login-node/versions.tf | 2 +- community/modules/scheduler/htcondor-configure/versions.tf | 2 +- .../scheduler/schedmd-slurm-gcp-v5-controller/versions.tf | 2 +- .../modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf | 2 +- community/modules/scripts/wait-for-startup/versions.tf | 2 +- modules/compute/vm-instance/versions.tf | 4 ++-- modules/file-system/filestore/versions.tf | 4 ++-- modules/monitoring/dashboard/versions.tf | 2 +- modules/network/pre-existing-vpc/versions.tf | 2 +- modules/scheduler/batch-login-node/versions.tf | 2 +- modules/scripts/startup-script/versions.tf | 2 +- 19 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index e00931b284..a487607d1f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -50,7 +50,7 @@ HPC deployments on the Google Cloud Platform.`, log.Fatalf("cmd.Help function failed: %s", err) } }, - Version: "v1.15.0", + Version: "v1.16.0", Annotations: annotation, } ) diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/versions.tf index 8462da4ed6..f41239efcc 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-node-group/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-node-group/v1.16.0" } required_version = ">= 0.13.0" } diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf b/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf index 0f4b57de97..ee889df477 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-partition/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-partition/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-partition/v1.16.0" } required_version = ">= 0.13.0" } diff --git a/community/modules/database/slurm-cloudsql-federation/versions.tf b/community/modules/database/slurm-cloudsql-federation/versions.tf index 38e80eb2ff..342a08a8be 100644 --- a/community/modules/database/slurm-cloudsql-federation/versions.tf +++ b/community/modules/database/slurm-cloudsql-federation/versions.tf @@ -30,10 +30,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.16.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:slurm-cloudsql-federation/v1.16.0" } required_version = ">= 0.13.0" diff --git a/community/modules/file-system/cloud-storage-bucket/versions.tf b/community/modules/file-system/cloud-storage-bucket/versions.tf index ab65f70f5d..6a48fcf157 100644 --- a/community/modules/file-system/cloud-storage-bucket/versions.tf +++ b/community/modules/file-system/cloud-storage-bucket/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:cloud-storage-bucket/v1.16.0" } required_version = ">= 0.14.0" } diff --git a/community/modules/file-system/nfs-server/versions.tf b/community/modules/file-system/nfs-server/versions.tf index 5c54ad0a43..abf9183c13 100644 --- a/community/modules/file-system/nfs-server/versions.tf +++ b/community/modules/file-system/nfs-server/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:nfs-server/v1.16.0" } required_version = ">= 0.14.0" diff --git a/community/modules/project/service-enablement/versions.tf b/community/modules/project/service-enablement/versions.tf index b0bfa9dadf..082292bd57 100644 --- a/community/modules/project/service-enablement/versions.tf +++ b/community/modules/project/service-enablement/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:service-enablement/v1.16.0" } required_version = ">= 0.14.0" diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/versions.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/versions.tf index aab119b8de..9fd2c48f4f 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/versions.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-controller/versions.tf @@ -16,7 +16,7 @@ terraform { provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-controller/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-controller/v1.16.0" } required_version = ">= 0.14.0" diff --git a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/versions.tf b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/versions.tf index 062f3fa746..2d8ab7aa5d 100644 --- a/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/versions.tf +++ b/community/modules/scheduler/SchedMD-slurm-on-gcp-login-node/versions.tf @@ -16,7 +16,7 @@ terraform { provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-login-node/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:SchedMD-slurm-on-gcp-login-node/v1.16.0" } required_version = ">= 0.14.0" diff --git a/community/modules/scheduler/htcondor-configure/versions.tf b/community/modules/scheduler/htcondor-configure/versions.tf index 4033126c56..2873f73a66 100644 --- a/community/modules/scheduler/htcondor-configure/versions.tf +++ b/community/modules/scheduler/htcondor-configure/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:htcondor-configure/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:htcondor-configure/v1.16.0" } required_version = ">= 0.13.0" diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf index 71fb304d37..434bb5c676 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-controller/v1.16.0" } required_version = ">= 0.14.0" } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf index f63f59e9bf..232cd9a79b 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:schedmd-slurm-gcp-v5-login/v1.16.0" } required_version = ">= 0.14.0" } diff --git a/community/modules/scripts/wait-for-startup/versions.tf b/community/modules/scripts/wait-for-startup/versions.tf index 15a1f96004..bee8dc44cb 100644 --- a/community/modules/scripts/wait-for-startup/versions.tf +++ b/community/modules/scripts/wait-for-startup/versions.tf @@ -26,7 +26,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:wait-for-startup/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/compute/vm-instance/versions.tf b/modules/compute/vm-instance/versions.tf index e0322e8473..829f9e8a48 100644 --- a/modules/compute/vm-instance/versions.tf +++ b/modules/compute/vm-instance/versions.tf @@ -27,10 +27,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.16.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/file-system/filestore/versions.tf b/modules/file-system/filestore/versions.tf index ac0b8ba420..c07d811c65 100644 --- a/modules/file-system/filestore/versions.tf +++ b/modules/file-system/filestore/versions.tf @@ -26,10 +26,10 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.16.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:filestore/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/monitoring/dashboard/versions.tf b/modules/monitoring/dashboard/versions.tf index ba89c80b4a..e506d7d67e 100644 --- a/modules/monitoring/dashboard/versions.tf +++ b/modules/monitoring/dashboard/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:dashboard/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/network/pre-existing-vpc/versions.tf b/modules/network/pre-existing-vpc/versions.tf index 8178c459fc..7efd3bcdb6 100644 --- a/modules/network/pre-existing-vpc/versions.tf +++ b/modules/network/pre-existing-vpc/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:pre-existing-vpc/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/scheduler/batch-login-node/versions.tf b/modules/scheduler/batch-login-node/versions.tf index b4d802c57c..9c71353364 100644 --- a/modules/scheduler/batch-login-node/versions.tf +++ b/modules/scheduler/batch-login-node/versions.tf @@ -22,7 +22,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:batch-login-node/v1.16.0" } required_version = ">= 0.14.0" diff --git a/modules/scripts/startup-script/versions.tf b/modules/scripts/startup-script/versions.tf index 8c1479ac66..4573ae149c 100644 --- a/modules/scripts/startup-script/versions.tf +++ b/modules/scripts/startup-script/versions.tf @@ -30,7 +30,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.15.0" + module_name = "blueprints/terraform/hpc-toolkit:startup-script/v1.16.0" } required_version = ">= 0.14.0"