Merge pull request #1147 from GoogleCloudPlatform/release-candidate

Release v1.16.0

.pre-commit-config.yaml

@@ -15,7 +15,7 @@
 ---
 repos:
 - repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.77.0
+  rev: v1.77.1
   hooks:
   - id: terraform_fmt
   - id: terraform_tflint
@@ -66,7 +66,7 @@ repos:
   hooks:
   - id: go-critic
     args: [-disable, "#experimental,sloppyTypeAssert"]
-- repo: https://github.com/ansible/ansible-lint.git
+- repo: https://github.com/ansible/ansible-lint
   rev: v6.11.0
   hooks:
   - id: ansible-lint
@@ -75,13 +75,13 @@ repos:
     types: [yaml]
     additional_dependencies:
     - ansible==6.*
-- repo: https://github.com/adrienverge/yamllint.git
+- repo: https://github.com/adrienverge/yamllint
   rev: v1.29.0
   hooks:
   - id: yamllint
     args: [-c=.yamllint]
 - repo: https://github.com/jackdewinter/pymarkdown
-  rev: v0.9.8
+  rev: v0.9.9
   hooks:
   - id: pymarkdown
     # Rules at https://github.com/jackdewinter/pymarkdown/tree/main/docs/rules

cmd/create.go

@@ -41,8 +41,8 @@ func init() {
 		"Sets the output directory where the HPC deployment directory will be created.")
 	createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars)
 	createCmd.Flags().StringSliceVar(&cliBEConfigVars, "backend-config", nil, msgCLIBackendConfig)
-	createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING",
-		validationLevelDesc)
+	createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", validationLevelDesc)
+	createCmd.Flags().StringSliceVar(&validatorsToSkip, "skip-validators", nil, skipValidatorsDesc)
 	createCmd.Flags().BoolVarP(&overwriteDeployment, "overwrite-deployment", "w", false,
 		"If specified, an existing deployment directory is overwritten by the new deployment. \n"+
 			"Note: Terraform state IS preserved. \n"+
@@ -60,7 +60,10 @@ var (
 	overwriteDeployment bool
 	validationLevel     string
 	validationLevelDesc = "Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")"
-	createCmd           = &cobra.Command{
+	validatorsToSkip    []string
+	skipValidatorsDesc  = "Validators to skip"
+
+	createCmd = &cobra.Command{
 		Use:   "create BLUEPRINT_NAME",
 		Short: "Create a new deployment.",
 		Long:  "Create a new deployment based on a provided blueprint.",
@@ -92,6 +95,9 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 	if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil {
 		log.Fatal(err)
 	}
+	if err := skipValidators(&deploymentConfig); err != nil {
+		log.Fatal(err)
+	}
 	if err := deploymentConfig.ExpandConfig(); err != nil {
 		log.Fatal(err)
 	}
@@ -105,3 +111,15 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 		}
 	}
 }
+
+func skipValidators(dc *config.DeploymentConfig) error {
+	if validatorsToSkip == nil {
+		return nil
+	}
+	for _, v := range validatorsToSkip {
+		if err := dc.SkipValidator(v); err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmd/expand.go

@@ -33,8 +33,8 @@ func init() {
 		"Output file for the expanded HPC Environment Definition.")
 	expandCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars)
 	expandCmd.Flags().StringSliceVar(&cliBEConfigVars, "backend-config", nil, msgCLIBackendConfig)
-	expandCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING",
-		validationLevelDesc)
+	expandCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", validationLevelDesc)
+	expandCmd.Flags().StringSliceVar(&validatorsToSkip, "skip-validators", nil, skipValidatorsDesc)
 	rootCmd.AddCommand(expandCmd)
 }
 
@@ -72,6 +72,9 @@ func runExpandCmd(cmd *cobra.Command, args []string) {
 	if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil {
 		log.Fatal(err)
 	}
+	if err := skipValidators(&deploymentConfig); err != nil {
+		log.Fatal(err)
+	}
 	if err := deploymentConfig.ExpandConfig(); err != nil {
 		log.Fatal(err)
 	}

cmd/root.go

@@ -50,7 +50,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.15.0",
+		Version:     "v1.16.0",
 		Annotations: annotation,
 	}
 )

community/examples/AMD/hpc-cluster-amd-slurmv5.yaml

@@ -70,7 +70,7 @@ deployment_groups:
           packages:
             slurm:
               externals:
-                - spec: slurm@22-05-3
+                - spec: slurm@22-05-8
                   prefix: /usr/local
               buildable: False
       - type: file
@@ -169,7 +169,7 @@ deployment_groups:
         # these images must match the images used by Slurm modules below because
         # we are building OpenMPI with PMI support in libaries contained in
         # Slurm installation
-        family: schedmd-v5-slurm-22-05-6-hpc-centos-7
+        family: schedmd-v5-slurm-22-05-8-hpc-centos-7
         project: schedmd-slurm-public
 
   - id: low_cost_node_group

community/examples/htcondor-pool.yaml

@@ -56,7 +56,8 @@ deployment_groups:
     - network1
     - htcondor_startup_central_manager
     settings:
-      name_prefix: central-manager
+      name_prefix: cm
+      add_deployment_name_before_prefix: true
       machine_type: c2-standard-4
       disable_public_ips: true
       service_account:
@@ -140,7 +141,8 @@ deployment_groups:
     - network1
     - htcondor_startup_access_point
     settings:
-      name_prefix: access-point
+      name_prefix: ap
+      add_deployment_name_before_prefix: true
       machine_type: c2-standard-4
       service_account:
         email: $(htcondor_configure.access_point_service_account)

community/examples/quantum-circuit-simulator.yaml

@@ -20,8 +20,8 @@ blueprint_name: quantum-circuit
 vars:
   project_id:  ## Set project id here
   deployment_name: qsim-demo
-  region: us-west4
-  zone: us-west4-b
+  region: us-central1
+  zone: us-central1-f
 
 # Documentation for each of the modules used below can be found at
 # https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
@@ -125,7 +125,10 @@ deployment_groups:
     - network1
     - quantum-simulator-setup
     settings:
-      machine_type: a2-highgpu-1g
+      machine_type: n1-standard-32
+      guest_accelerator:
+      - type: nvidia-tesla-t4
+        count: 1
       instance_image:
         project: ubuntu-os-cloud
         family: ubuntu-2004-lts

community/examples/slurm-gcp-v5-ubuntu2004.yaml

@@ -22,7 +22,9 @@ vars:
   region: us-central1
   zone: us-central1-c
   instance_image:
-    family: schedmd-v5-slurm-22-05-6-ubuntu-2004-lts
+    # Please refer to the following link for the latest images:
+    # https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems
+    family: schedmd-v5-slurm-22-05-8-ubuntu-2004-lts
     project: projects/schedmd-slurm-public/global/images/family
 
 

community/front-end/ofe/infrastructure_files/gcs_bucket/webserver/startup.sh

@@ -192,7 +192,7 @@ sudo su - gcluster -c /bin/bash <<EOF
   printf "\nInitalising Django environments...\n"
   mkdir /opt/gcluster/run
   pushd website
-  python manage.py makemigrations
+  python manage.py makemigrations ghpcfe
   python manage.py migrate
   printf "\nCreating django super user..."
   DJANGO_SUPERUSER_PASSWORD=$DJANGO_PASSWORD python manage.py createsuperuser --username $DJANGO_USERNAME --email $DJANGO_EMAIL --noinput

community/front-end/ofe/requirements.txt

@@ -1,72 +1,84 @@
-archspec==0.1.3
-argcomplete==2.0.0
-asgiref==3.5.0
-astroid==2.15.0
-cachetools==5.0.0
+archspec==0.2.0
+argcomplete==3.0.5
+asgiref==3.6.0
+astroid==2.15.1
+backports.zoneinfo==0.2.1
+cachetools==5.3.0
 certifi==2022.12.7
-cffi==1.15.0
-charset-normalizer==2.0.12
-click==7.1.2
-cryptography==39.0.1
+cffi==1.15.1
+cfgv==3.3.1
+charset-normalizer==3.1.0
+click==8.1.3
+cryptography==40.0.1
+decorator==5.1.1
 defusedxml==0.7.1
 dill==0.3.6
-Django==3.2.18
-django-allauth==0.48.0
-django-extensions==3.1.5
-# Need version 0.11.0 to be released with fixes for Django 3.2
+distlib==0.3.6
+# django-revproxy==0.11.0 released but not yet in pypi
 git+https://github.com/jazzband/django-revproxy.git@d2234005135dc0771b7c4e0bb0465664ccfa5787
-djangorestframework==3.13.1
-google-api-core==2.5.0
-google-api-python-client==2.37.0
-google-auth==2.6.0
+Django==4.1.7
+django-allauth==0.54.0
+django-extensions==3.2.1
+djangorestframework==3.14.0
+filelock==3.10.7
+google-api-core==2.11.0
+google-api-python-client==2.83.0
+google-auth==2.17.1
 google-auth-httplib2==0.1.0
-google-cloud-billing==1.4.1
+google-cloud-billing==1.10.1
 google-cloud-core==2.3.2
-google-cloud-pubsub==2.9.0
-google-cloud-storage==2.1.0
-google-crc32c==1.3.0
-google-resumable-media==2.2.1
-googleapis-common-protos==1.54.0
-grafana_api==1.0.3
-grpc-google-iam-v1==0.12.3
-grpcio==1.43.0
-grpcio-status==1.43.0
-h11==0.13.0
-httplib2==0.20.4
-idna==3.3
+google-cloud-pubsub==2.15.2
+google-cloud-storage==2.8.0
+google-crc32c==1.5.0
+google-resumable-media==2.4.1
+googleapis-common-protos==1.59.0
+grafana-api==1.0.3
+grpc-google-iam-v1==0.12.6
+grpcio==1.53.0
+grpcio-status==1.53.0
+h11==0.14.0
+httplib2==0.22.0
+identify==2.5.22
+idna==3.4
 isort==5.12.0
-lazy-object-proxy==1.7.1
-libcst==0.4.1
-mccabe==0.6.1
-mypy-extensions==0.4.3
+lazy-object-proxy==1.9.0
+libcst==0.4.9
+mccabe==0.7.0
+mypy-extensions==1.0.0
+nodeenv==1.7.0
 oauthlib==3.2.2
-platformdirs==2.5.0
-pre-commit==2.17.0
+platformdirs==3.2.0
+pre-commit==3.2.1
 proto-plus==1.22.2
-protobuf==3.19.5
+protobuf==4.22.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pycparser==2.21
-PyJWT==2.4.0
-pylint==2.12.2
-pylint-django==2.5.0
+PyJWT==2.6.0
+pylint==2.17.1
+pylint-django==2.5.3
 pylint-plugin-utils==0.7
-pyparsing==3.0.7
+pyparsing==3.0.9
 python3-openid==3.2.0
-pytz==2021.3
+pytz==2023.3
 PyYAML==6.0
-retry==0.9.2
-requests==2.27.1
+requests==2.28.2
 requests-oauthlib==1.3.1
-rsa==4.8
+retry==0.9.2
+rsa==4.9
+semantic-version==2.10.0
+setuptools-rust==1.5.2
 six==1.16.0
-sqlparse==0.4.2
+sqlparse==0.4.3
 toml==0.10.2
-typing-inspect==0.7.1
-typing_extensions==4.1.1
+tomli==2.0.1
+tomlkit==0.11.7
+typing-inspect==0.8.0
+typing_extensions==4.5.0
 uritemplate==4.1.1
-urllib3==1.26.8
-uvicorn==0.17.4
-wrapt==1.13.3
-xmltodict==0.12.0
-yq==2.13.0
+urllib3==1.26.15
+uvicorn==0.21.1
+virtualenv==20.21.0
+wrapt==1.15.0
+xmltodict==0.13.0
+yq==3.1.1

community/front-end/ofe/website/ghpcfe/cluster_manager/filesystem.py

@@ -56,7 +56,7 @@ def write_filestore_yaml(fs: GCPFilestoreFilesystem, target_dir: Path) -> None:
     id: {fs.name}
     settings:
       filestore_share_name: {export_name[1:]}
-      network_name: {fs.vpc.cloud_id}
+      network_id: projects/{project_id}/global/networks/{fs.vpc.cloud_id}
       zone: {fs.cloud_zone}
       size_gb: {fs.capacity}
       filestore_tier: {fs.get_performance_tier_display()}

community/front-end/ofe/website/ghpcfe/signals.py

@@ -14,14 +14,14 @@
 
 """Signal handlers for model state"""
 
-from django.db.models.signals import pre_save, post_delete
+from django.db.models.signals import pre_save, post_delete, post_save
 from django.dispatch import receiver
 from .models import Cluster, VirtualNetwork
 
 # Pylint misses the sender decorator behaviour here
 #pylint: disable=unused-argument
 
-@receiver(pre_save, sender=VirtualNetwork)
+@receiver(post_save, sender=VirtualNetwork)
 def sync_vnet_subnet_state(sender, **kwargs):
     vpc = kwargs["instance"]
     for sn in vpc.subnets.all():

community/front-end/ofe/website/ghpcfe/views/grafana.py

@@ -25,11 +25,12 @@ class GrafanaProxyView(LoginRequiredMixin, ProxyView):
     def get_proxy_request_headers(self, request):
         headers = super().get_proxy_request_headers(request)
         headers["X-WEBAUTH-USER"] = request.user.email
+        headers["Host"] = request.get_host()
         return headers
 
     def dispatch(self, request, path):
         response = super().dispatch(request, path)
-        response.headers["X-Frame-Options"] = "SAMEORIGIN"
+        response["X-Frame-Options"] = "SAMEORIGIN"
         return response
 
 class GrafanaView(LoginRequiredMixin, base.TemplateView):

community/modules/compute/SchedMD-slurm-on-gcp-partition/README.md

@@ -74,10 +74,10 @@ No resources.
 | <a name="input_image_hyperthreads"></a> [image\_hyperthreads](#input\_image\_hyperthreads) | Enable hyperthreading | `bool` | `false` | no |
 | <a name="input_instance_image"></a> [instance\_image](#input\_instance\_image) | Defines the image that will be used by the compute VMs in this partition.<br>Expected Fields:<br>name: The name of the image. Mutually exclusive with family.<br>family: The image family to use. Mutually exclusive with name.<br>project: The project where the image is hosted.<br>Custom images must comply with Slurm on GCP requirements. | `map(string)` | <pre>{<br>  "family": "schedmd-slurm-21-08-8-hpc-centos-7",<br>  "project": "schedmd-slurm-public"<br>}</pre> | no |
 | <a name="input_instance_template"></a> [instance\_template](#input\_instance\_template) | Instance template to use to create partition instances | `string` | `null` | no |
-| <a name="input_labels"></a> [labels](#input\_labels) | Labels to add to partition compute instances. List of key key, value pairs. | `any` | `{}` | no |
+| <a name="input_labels"></a> [labels](#input\_labels) | Labels to add to partition compute instances. Key-value pairs. | `map(string)` | `{}` | no |
 | <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | Compute Platform machine type to use for this partition compute nodes | `string` | `"c2-standard-60"` | no |
 | <a name="input_max_node_count"></a> [max\_node\_count](#input\_max\_node\_count) | Maximum number of nodes allowed in this partition | `number` | `50` | no |
-| <a name="input_network_storage"></a> [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | <pre>list(object({<br>    server_ip     = string,<br>    remote_mount  = string,<br>    local_mount   = string,<br>    fs_type       = string,<br>    mount_options = string<br>  }))</pre> | `[]` | no |
+| <a name="input_network_storage"></a> [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured on the partition compute nodes. | <pre>list(object({<br>    server_ip             = string,<br>    remote_mount          = string,<br>    local_mount           = string,<br>    fs_type               = string,<br>    mount_options         = string,<br>    client_install_runner = map(string)<br>    mount_runner          = map(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_partition_name"></a> [partition\_name](#input\_partition\_name) | The name of the slurm partition | `string` | n/a | yes |
 | <a name="input_preemptible_bursting"></a> [preemptible\_bursting](#input\_preemptible\_bursting) | Should use preemptibles to burst | `string` | `false` | no |
 | <a name="input_regional_capacity"></a> [regional\_capacity](#input\_regional\_capacity) | If True, then create instances in the region that has available capacity. Specify the region in the zone field. | `bool` | `false` | no |