diff --git a/community/modules/compute/htcondor-execute-point/gpu_definition.tf b/community/modules/compute/htcondor-execute-point/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/compute/htcondor-execute-point/gpu_definition.tf +++ b/community/modules/compute/htcondor-execute-point/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/compute/pbspro-execution/README.md b/community/modules/compute/pbspro-execution/README.md index 4fa927cbee..a47da78f09 100644 --- a/community/modules/compute/pbspro-execution/README.md +++ b/community/modules/compute/pbspro-execution/README.md @@ -102,7 +102,7 @@ No resources. | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation | `string` | `"c2-standard-60"` | no | | [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | | [name\_prefix](#input\_name\_prefix) | Name prefix for PBS execution hostnames | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface

**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.

Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | | [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | diff --git a/community/modules/compute/pbspro-execution/variables.tf b/community/modules/compute/pbspro-execution/variables.tf index fa818154b4..32ab77d2f0 100644 --- a/community/modules/compute/pbspro-execution/variables.tf +++ b/community/modules/compute/pbspro-execution/variables.tf @@ -203,7 +203,7 @@ variable "network_interfaces" { subnetwork (string, required if network is not supplied) subnetwork_project (string, optional) network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET"]) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) queue_count (number, optional) access_config (object, optional) diff --git a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/gpu_definition.tf b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v5-node-group/gpu_definition.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v5-node-group/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md index 1ee007d0a1..88948ef2e4 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/README.md @@ -74,7 +74,7 @@ modules. For support with the underlying modules, see the instructions in the | Name | Source | Version | |------|--------|---------| -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 6.8.5 | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | b0575ab | ## Resources diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/gpu_definition.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/gpu_definition.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf index bab7de7eaa..68c9703147 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset-dynamic/main.tf @@ -56,7 +56,7 @@ locals { } module "slurm_nodeset_template" { - source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=6.8.5" + source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=b0575ab" project_id = var.project_id region = var.region diff --git a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/gpu_definition.tf b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/gpu_definition.tf +++ b/community/modules/compute/schedmd-slurm-gcp-v6-nodeset/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/network/rdma-vpc/README.md b/community/modules/network/rdma-vpc/README.md new file mode 100644 index 0000000000..a534b362e0 --- /dev/null +++ b/community/modules/network/rdma-vpc/README.md @@ -0,0 +1,82 @@ +## Description + +This is an experimental VPC module. + +Documentation will be updated at a later point. + +## License + + +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.15.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [vpc](#module\_vpc) | ./vpc-submodule | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [allowed\_ssh\_ip\_ranges](#input\_allowed\_ssh\_ip\_ranges) | A list of CIDR IP ranges from which to allow ssh access | `list(string)` | `[]` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [deployment\_name](#input\_deployment\_name) | The name of the current deployment | `string` | n/a | yes | +| [enable\_iap\_rdp\_ingress](#input\_enable\_iap\_rdp\_ingress) | Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels | `bool` | `false` | no | +| [enable\_iap\_ssh\_ingress](#input\_enable\_iap\_ssh\_ingress) | Enable a firewall rule to allow SSH access using IAP tunnels | `bool` | `true` | no | +| [enable\_iap\_winrm\_ingress](#input\_enable\_iap\_winrm\_ingress) | Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels | `bool` | `false` | no | +| [enable\_internal\_traffic](#input\_enable\_internal\_traffic) | Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network | `bool` | `true` | no | +| [extra\_iap\_ports](#input\_extra\_iap\_ports) | A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable\_iap variables for standard ports) | `list(string)` | `[]` | no | +| [firewall\_log\_config](#input\_firewall\_log\_config) | Firewall log configuration for Toolkit firewall rules (var.enable\_iap\_ssh\_ingress and others) | `string` | `"DISABLE_LOGGING"` | no | +| [firewall\_rules](#input\_firewall\_rules) | List of firewall rules | `any` | `[]` | no | +| [mtu](#input\_mtu) | The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `8896` | no | +| [network\_address\_range](#input\_network\_address\_range) | IP address range (CIDR) for global network | `string` | `"10.0.0.0/9"` | no | +| [network\_description](#input\_network\_description) | An optional description of this resource (changes will trigger resource destroy/create) | `string` | `""` | no | +| [network\_name](#input\_network\_name) | The name of the network to be created (if unsupplied, will default to "{deployment\_name}-net") | `string` | `null` | no | +| [network\_profile](#input\_network\_profile) | Profile name for VPC configuration | `string` | `null` | no | +| [network\_routing\_mode](#input\_network\_routing\_mode) | The network routing mode (default "GLOBAL") | `string` | `"GLOBAL"` | no | +| [nic\_type](#input\_nic\_type) | NIC type for use in modules that use the output | `string` | `"MRDMA"` | no | +| [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | +| [region](#input\_region) | The default region for Cloud resources | `string` | n/a | yes | +| [secondary\_ranges](#input\_secondary\_ranges) | Secondary ranges that will be used in some of the subnets. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions. | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnetworks\_template](#input\_subnetworks\_template) | Rules for creating subnetworks within the VPC | 
object({
    count          = number
    name_prefix    = string
    ip_range       = string
    region         = string
    private_access = optional(bool)
  })
| 
{
  "count": 8,
  "ip_range": "192.168.0.0/16",
  "name_prefix": "subnet",
  "region": null
}
| no | + +## Outputs + +| Name | Description | +|------|-------------| +| [network\_id](#output\_network\_id) | ID of the new VPC network | +| [network\_name](#output\_network\_name) | Name of the new VPC network | +| [network\_self\_link](#output\_network\_self\_link) | Self link of the new VPC network | +| [subnetwork\_interfaces](#output\_subnetwork\_interfaces) | Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance) | +| [subnetwork\_interfaces\_gke](#output\_subnetwork\_interfaces\_gke) | Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool) | +| [subnetwork\_name\_prefix](#output\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | +| [subnetworks](#output\_subnetworks) | Full list of subnetwork objects belonging to the new VPC network | + diff --git a/community/modules/network/rdma-vpc/main.tf b/community/modules/network/rdma-vpc/main.tf new file mode 100644 index 0000000000..71e764a2b6 --- /dev/null +++ b/community/modules/network/rdma-vpc/main.tf @@ -0,0 +1,159 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +locals { + autoname = replace(var.deployment_name, "_", "-") + network_name = var.network_name == null ? "${local.autoname}-net" : var.network_name + + new_bits = ceil(log(var.subnetworks_template.count, 2)) + template_subnetworks = [for i in range(var.subnetworks_template.count) : + { + subnet_name = "${var.subnetworks_template.name_prefix}-${i}" + subnet_region = try(var.subnetworks_template.region, var.region) + subnet_ip = cidrsubnet(var.subnetworks_template.ip_range, local.new_bits, i) + subnet_private_access = coalesce(var.subnetworks_template.private_access, false) + } + ] + + iap_ports = distinct(concat(compact([ + var.enable_iap_rdp_ingress ? "3389" : "", + var.enable_iap_ssh_ingress ? "22" : "", + var.enable_iap_winrm_ingress ? "5986" : "", + ]), var.extra_iap_ports)) + + firewall_log_api_values = { + "DISABLE_LOGGING" = null + "INCLUDE_ALL_METADATA" = { metadata = "INCLUDE_ALL_METADATA" }, + "EXCLUDE_ALL_METADATA" = { metadata = "EXCLUDE_ALL_METADATA" }, + } + firewall_log_config = lookup(local.firewall_log_api_values, var.firewall_log_config, null) + + allow_iap_ingress = { + name = "${local.network_name}-fw-allow-iap-ingress" + description = "allow TCP access via Identity-Aware Proxy" + direction = "INGRESS" + priority = null + ranges = ["35.235.240.0/20"] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = local.iap_ports + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_ssh_ingress = { + name = "${local.network_name}-fw-allow-ssh-ingress" + description = "allow SSH access" + direction = "INGRESS" + priority = null + ranges = var.allowed_ssh_ip_ranges + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["22"] + }] + deny = [] + log_config = local.firewall_log_config + } + + allow_internal_traffic = { + name = "${local.network_name}-fw-allow-internal-traffic" + priority = null + description = "allow traffic between nodes of this VPC" + direction = "INGRESS" + ranges = [var.network_address_range] + source_tags = null + source_service_accounts = null + target_tags = null + target_service_accounts = null + allow = [{ + protocol = "tcp" + ports = ["0-65535"] + }, { + protocol = "udp" + ports = ["0-65535"] + }, { + protocol = "icmp" + ports = null + }, + ] + deny = [] + log_config = local.firewall_log_config + } + + firewall_rules = concat( + var.firewall_rules, + length(var.allowed_ssh_ip_ranges) > 0 ? [local.allow_ssh_ingress] : [], + var.enable_internal_traffic ? [local.allow_internal_traffic] : [], + length(local.iap_ports) > 0 ? [local.allow_iap_ingress] : [] + ) + + url_parts = split("/", var.network_profile) + profile_name = upper(element(local.url_parts, length(local.url_parts) - 1)) + output_subnets = [ + for subnet in module.vpc.subnets : { + network = null + subnetwork = subnet.self_link + subnetwork_project = null # will populate from subnetwork_self_link + network_ip = null + nic_type = var.nic_type + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] + + output_subnets_gke = [ + for i in range(length(module.vpc.subnets)) : { + network = local.network_name + subnetwork = local.template_subnetworks[i].subnet_name + subnetwork_project = var.project_id + network_ip = "" + nic_type = coalesce(var.nic_type, try(regex("IRDMA", local.profile_name), regex("MRDMA", local.profile_name), "RDMA")) + stack_type = null + queue_count = null + access_config = [] + ipv6_access_config = [] + alias_ip_range = [] + } + ] +} + +module "vpc" { + source = "./vpc-submodule" + network_name = local.network_name + project_id = var.project_id + auto_create_subnetworks = false + subnets = local.template_subnetworks + secondary_ranges = var.secondary_ranges + routing_mode = var.network_routing_mode + mtu = var.mtu + description = var.network_description + shared_vpc_host = var.shared_vpc_host + delete_default_internet_gateway_routes = var.delete_default_internet_gateway_routes + firewall_rules = local.firewall_rules + network_profile = var.network_profile +} diff --git a/community/modules/network/rdma-vpc/metadata.yaml b/community/modules/network/rdma-vpc/metadata.yaml new file mode 100644 index 0000000000..4c2f23a8d7 --- /dev/null +++ b/community/modules/network/rdma-vpc/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/community/modules/network/rdma-vpc/outputs.tf b/community/modules/network/rdma-vpc/outputs.tf new file mode 100644 index 0000000000..1c2a304fd8 --- /dev/null +++ b/community/modules/network/rdma-vpc/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +output "network_name" { + description = "Name of the new VPC network" + value = module.vpc.network_name + depends_on = [module.vpc] +} + +output "network_id" { + description = "ID of the new VPC network" + value = module.vpc.network_id + depends_on = [module.vpc] +} + +output "network_self_link" { + description = "Self link of the new VPC network" + value = module.vpc.network_self_link + depends_on = [module.vpc] +} + +output "subnetworks" { + description = "Full list of subnetwork objects belonging to the new VPC network" + value = module.vpc.subnets + depends_on = [module.vpc] +} + +output "subnetwork_interfaces" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with vm-instance)" + value = local.output_subnets + depends_on = [module.vpc] +} + +# The output subnetwork_interfaces is compatible with vm-instance module but not with gke-node-pool +# See https://github.com/GoogleCloudPlatform/cluster-toolkit/blob/99493df21cecf6a092c45298bf7a45e0343cf622/modules/compute/vm-instance/variables.tf#L220 +# So, we need a separate output that makes the network and subnetwork names available +output "subnetwork_interfaces_gke" { + description = "Full list of subnetwork objects belonging to the new VPC network (compatible with gke-node-pool)" + value = local.output_subnets_gke + depends_on = [module.vpc] +} + +output "subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + value = var.subnetworks_template.name_prefix +} diff --git a/community/modules/network/rdma-vpc/variables.tf b/community/modules/network/rdma-vpc/variables.tf new file mode 100644 index 0000000000..a5ba44d455 --- /dev/null +++ b/community/modules/network/rdma-vpc/variables.tf @@ -0,0 +1,183 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +variable "project_id" { + description = "Project in which the HPC deployment will be created" + type = string +} + +variable "network_name" { + description = "The name of the network to be created (if unsupplied, will default to \"{deployment_name}-net\")" + type = string + default = null +} + +variable "region" { + description = "The default region for Cloud resources" + type = string +} + +variable "deployment_name" { + description = "The name of the current deployment" + type = string +} + +variable "network_address_range" { + description = "IP address range (CIDR) for global network" + type = string + default = "10.0.0.0/9" + + validation { + condition = can(cidrhost(var.network_address_range, 0)) + error_message = "IP address range must be in CIDR format." + } +} + +variable "mtu" { + type = number + description = "The network MTU (default: 8896). Recommended values: 0 (use Compute Engine default), 1460 (default outside HPC environments), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 8896 +} + +variable "subnetworks_template" { + # TODO: Add validation and improve description + description = "Rules for creating subnetworks within the VPC" + type = object({ + count = number + name_prefix = string + ip_range = string + region = string + private_access = optional(bool) + }) + default = { + count = 8 + name_prefix = "subnet" + ip_range = "192.168.0.0/16" + region = null + } +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = "Secondary ranges that will be used in some of the subnets. Please see https://goo.gle/hpc-toolkit-vpc-deprecation for migration instructions." + default = {} +} + +variable "network_routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default \"GLOBAL\")" + + validation { + condition = contains(["GLOBAL", "REGIONAL"], var.network_routing_mode) + error_message = "The network routing mode must either be \"GLOBAL\" or \"REGIONAL\"." + } +} + +variable "network_description" { + type = string + description = "An optional description of this resource (changes will trigger resource destroy/create)" + default = "" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + +variable "enable_iap_ssh_ingress" { + type = bool + description = "Enable a firewall rule to allow SSH access using IAP tunnels" + default = true +} + +variable "enable_iap_rdp_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Desktop Protocol access using IAP tunnels" + default = false +} + +variable "enable_iap_winrm_ingress" { + type = bool + description = "Enable a firewall rule to allow Windows Remote Management (WinRM) access using IAP tunnels" + default = false +} + +variable "enable_internal_traffic" { + type = bool + description = "Enable a firewall rule to allow all internal TCP, UDP, and ICMP traffic within the network" + default = true +} + +variable "extra_iap_ports" { + type = list(string) + description = "A list of TCP ports for which to create firewall rules that enable IAP for TCP forwarding (use dedicated enable_iap variables for standard ports)" + default = [] +} + +variable "allowed_ssh_ip_ranges" { + type = list(string) + description = "A list of CIDR IP ranges from which to allow ssh access" + default = [] + + validation { + condition = alltrue([for r in var.allowed_ssh_ip_ranges : can(cidrhost(r, 32))]) + error_message = "Each element of var.allowed_ssh_ip_ranges must be a valid CIDR-formatted IPv4 range." + } +} + +variable "firewall_rules" { + type = any + description = "List of firewall rules" + default = [] +} + +variable "firewall_log_config" { + type = string + description = "Firewall log configuration for Toolkit firewall rules (var.enable_iap_ssh_ingress and others)" + default = "DISABLE_LOGGING" + nullable = false + + validation { + condition = contains([ + "INCLUDE_ALL_METADATA", + "EXCLUDE_ALL_METADATA", + "DISABLE_LOGGING", + ], var.firewall_log_config) + error_message = "var.firewall_log_config must be set to \"DISABLE_LOGGING\", or enable logging with \"INCLUDE_ALL_METADATA\" or \"EXCLUDE_ALL_METADATA\"" + } +} + +variable "network_profile" { + # TODO Update this description + description = "Profile name for VPC configuration" + type = string + default = null +} + +variable "nic_type" { + description = "NIC type for use in modules that use the output" + type = string + nullable = false + default = "MRDMA" +} diff --git a/community/modules/network/rdma-vpc/versions.tf b/community/modules/network/rdma-vpc/versions.tf new file mode 100644 index 0000000000..71b7106734 --- /dev/null +++ b/community/modules/network/rdma-vpc/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +terraform { + required_version = ">= 0.15.0" +} diff --git a/community/modules/network/rdma-vpc/vpc-submodule/README.md b/community/modules/network/rdma-vpc/vpc-submodule/README.md new file mode 100644 index 0000000000..253ebaf772 --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/README.md @@ -0,0 +1,163 @@ +# Terraform Network Module + +This is a temporary sub-module to use the new experimental features in the VPC module: network_profile + + +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3 | +| [google](#requirement\_google) | >= 4.64 | +| [google-beta](#requirement\_google-beta) | >= 6.13.0 | + +## Providers + +| Name | Version | +|------|---------| +| [google](#provider\_google) | >= 4.64 | +| [google-beta](#provider\_google-beta) | >= 6.13.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firewall\_rules](#module\_firewall\_rules) | terraform-google-modules/network/google//modules/firewall-rules | ~> 9.3 | +| [routes](#module\_routes) | terraform-google-modules/network/google//modules/routes | ~> 9.3 | +| [subnets](#module\_subnets) | terraform-google-modules/network/google//modules/subnets | ~> 9.3 | + +## Resources + +| Name | Type | +|------|------| +| [google-beta_google_compute_network.network](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_network) | resource | +| [google_compute_shared_vpc_host_project.shared_vpc_host](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_shared_vpc_host_project) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_create\_subnetworks](#input\_auto\_create\_subnetworks) | When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources. | `bool` | `false` | no | +| [delete\_default\_internet\_gateway\_routes](#input\_delete\_default\_internet\_gateway\_routes) | If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted | `bool` | `false` | no | +| [description](#input\_description) | An optional description of this resource. The resource must be recreated to modify this field. | `string` | `""` | no | +| [egress\_rules](#input\_egress\_rules) | List of egress rules. This will be ignored if variable 'rules' is non-empty | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [enable\_ipv6\_ula](#input\_enable\_ipv6\_ula) | Enabled IPv6 ULA, this is a permanent change and cannot be undone! (default 'false') | `bool` | `false` | no | +| [firewall\_rules](#input\_firewall\_rules) | This is DEPRECATED and available for backward compatibility. Use ingress\_rules and egress\_rules variables. List of firewall rules | 
list(object({
    name                    = string
    description             = optional(string, null)
    direction               = optional(string, "INGRESS")
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    ranges                  = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules. This will be ignored if variable 'rules' is non-empty | 
list(object({
    name                    = string
    description             = optional(string, null)
    disabled                = optional(bool, null)
    priority                = optional(number, null)
    destination_ranges      = optional(list(string), [])
    source_ranges           = optional(list(string), [])
    source_tags             = optional(list(string))
    source_service_accounts = optional(list(string))
    target_tags             = optional(list(string))
    target_service_accounts = optional(list(string))

    allow = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    deny = optional(list(object({
      protocol = string
      ports    = optional(list(string))
    })), [])
    log_config = optional(object({
      metadata = string
    }))
  }))
| `[]` | no | +| [internal\_ipv6\_range](#input\_internal\_ipv6\_range) | When enabling IPv6 ULA, optionally, specify a /48 from fd20::/20 (default null) | `string` | `null` | no | +| [mtu](#input\_mtu) | The network MTU (If set to 0, meaning MTU is unset - defaults to '1460'). Recommended values: 1460 (default for historic reasons), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively. | `number` | `0` | no | +| [network\_firewall\_policy\_enforcement\_order](#input\_network\_firewall\_policy\_enforcement\_order) | Set the order that Firewall Rules and Firewall Policies are evaluated. Valid values are `BEFORE_CLASSIC_FIREWALL` and `AFTER_CLASSIC_FIREWALL`. (default null or equivalent to `AFTER_CLASSIC_FIREWALL`) | `string` | `null` | no | +| [network\_name](#input\_network\_name) | The name of the network being created | `string` | n/a | yes | +| [network\_profile](#input\_network\_profile) | Profile name for VPC configuration | `string` | `null` | no | +| [project\_id](#input\_project\_id) | The ID of the project where this VPC will be created | `string` | n/a | yes | +| [routes](#input\_routes) | List of routes being created in this VPC | `list(map(string))` | `[]` | no | +| [routing\_mode](#input\_routing\_mode) | The network routing mode (default 'GLOBAL') | `string` | `"GLOBAL"` | no | +| [secondary\_ranges](#input\_secondary\_ranges) | Secondary ranges that will be used in some of the subnets | `map(list(object({ range_name = string, ip_cidr_range = string })))` | `{}` | no | +| [shared\_vpc\_host](#input\_shared\_vpc\_host) | Makes this project a Shared VPC host if 'true' (default 'false') | `bool` | `false` | no | +| [subnets](#input\_subnets) | The list of subnets being created | 
list(object({
    subnet_name                      = string
    subnet_ip                        = string
    subnet_region                    = string
    subnet_private_access            = optional(string)
    subnet_private_ipv6_access       = optional(string)
    subnet_flow_logs                 = optional(string)
    subnet_flow_logs_interval        = optional(string)
    subnet_flow_logs_sampling        = optional(string)
    subnet_flow_logs_metadata        = optional(string)
    subnet_flow_logs_filter          = optional(string)
    subnet_flow_logs_metadata_fields = optional(list(string))
    description                      = optional(string)
    purpose                          = optional(string)
    role                             = optional(string)
    stack_type                       = optional(string)
    ipv6_access_type                 = optional(string)
  }))
| n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [network](#output\_network) | The created network | +| [network\_id](#output\_network\_id) | The ID of the VPC being created | +| [network\_name](#output\_network\_name) | The name of the VPC being created | +| [network\_self\_link](#output\_network\_self\_link) | The URI of the VPC being created | +| [project\_id](#output\_project\_id) | VPC project id | +| [route\_names](#output\_route\_names) | The route names associated with this VPC | +| [subnets](#output\_subnets) | A map with keys of form subnet\_region/subnet\_name and values being the outputs of the google\_compute\_subnetwork resources used to create corresponding subnets. | +| [subnets\_flow\_logs](#output\_subnets\_flow\_logs) | Whether the subnets will have VPC flow logs enabled | +| [subnets\_ids](#output\_subnets\_ids) | The IDs of the subnets being created | +| [subnets\_ips](#output\_subnets\_ips) | The IPs and CIDRs of the subnets being created | +| [subnets\_names](#output\_subnets\_names) | The names of the subnets being created | +| [subnets\_private\_access](#output\_subnets\_private\_access) | Whether the subnets will have access to Google API's without a public IP | +| [subnets\_regions](#output\_subnets\_regions) | The region where the subnets will be created | +| [subnets\_secondary\_ranges](#output\_subnets\_secondary\_ranges) | The secondary ranges associated with these subnets | +| [subnets\_self\_links](#output\_subnets\_self\_links) | The self-links of subnets being created | + + +### Subnet Inputs + +The subnets list contains maps, where each object represents a subnet. Each map has the following inputs (please see examples folder for additional references): + +| Name | Description | Type | Default | Required | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------- | :----: | :----------------------: | :------: | +| subnet\_name | The name of the subnet being created | string | - | yes | +| subnet\_ip | The IP and CIDR range of the subnet being created | string | - | yes | +| subnet\_region | The region where the subnet will be created | string | - | yes | +| subnet\_private\_access | Whether this subnet will have private Google access enabled | string | `"false"` | no | +| subnet\_private\_ipv6\_access| The private IPv6 google access type for the VMs in this subnet | string | - | no | +| subnet\_flow\_logs | Whether the subnet will record and send flow log data to logging | string | `"false"` | no | +| subnet\_flow\_logs\_interval | If subnet\_flow\_logs is true, sets the aggregation interval for collecting flow logs | string | `"INTERVAL_5_SEC"` | no | +| subnet\_flow\_logs\_sampling | If subnet\_flow\_logs is true, set the sampling rate of VPC flow logs within the subnetwork | string | `"0.5"` | no | +| subnet\_flow\_logs\_metadata | If subnet\_flow\_logs is true, configures whether metadata fields should be added to the reported VPC flow logs | string | `"INCLUDE_ALL_METADATA"` | no | +| subnet\_flow\_logs\_filter | Export filter defining which VPC flow logs should be logged, see https://cloud.google.com/vpc/docs/flow-logs#filtering for formatting details | string | `"true"` | no | +| subnet\_flow\_logs\_metadata\_fields | List of metadata fields that should be added to reported logs. Can only be specified if VPC flow logs for this subnetwork is enabled and "metadata" is set to CUSTOM_METADATA. | any | - | no | +| description | An optional description of this resource. Provide this property when you create the resource. This field can be set only at resource creation time | string | - | no | +| purpose | The purpose of the subnet usage. Whether it is to be used as a regular subnet or for proxy or loadbalacing purposes, see https://cloud.google.com/vpc/docs/subnets#purpose for more details | string | `"PRIVATE"` | no | +| role | The role of the subnet when using it as a proxy or loadbalancer network. Whether it is to be used as the active or as a backup subnet, see https://cloud.google.com/load-balancing/docs/proxy-only-subnets#proxy_only_subnet_create for more details | string | - | no | +| stack\_type | `IPV4_ONLY` or `IPV4_IPV6` for dual-stack networking | string | - | no | +| ipv6\_access\_type | `INTERNAL` or `EXTERNAL`. `INTERNAL` requires ULA be enabled on the VPC | string | - | no | + +### Route Inputs + +The routes list contains maps, where each object represents a route. For the next_hop_\* inputs, only one is possible to be used in each route. Having two next_hop_\* inputs will produce an error. Each map has the following inputs (please see examples folder for additional references): + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| name | The name of the route being created | string | - | no | +| description | The description of the route being created | string | - | no | +| tags | The network tags assigned to this route. This is a list in string format. Eg. "tag-01,tag-02"| string | - | yes | +| destination\_range | The destination range of outgoing packets that this route applies to. Only IPv4 is supported | string | - | yes +| next\_hop\_internet | Whether the next hop to this route will the default internet gateway. Use "true" to enable this as next hop | string | `"false"` | yes | +| next\_hop\_ip | Network IP address of an instance that should handle matching packets | string | - | yes | +| next\_hop\_instance | URL or name of an instance that should handle matching packets. If just name is specified "next\_hop\_instance\_zone" is required | string | - | yes | +| next\_hop\_instance\_zone | The zone of the instance specified in next\_hop\_instance. Only required if next\_hop\_instance is specified as a name | string | - | no | +| next\_hop\_vpn\_tunnel | URL to a VpnTunnel that should handle matching packets | string | - | yes | +| priority | The priority of this route. Priority is used to break ties in cases where there is more than one matching route of equal prefix length. In the case of two routes with equal prefix length, the one with the lowest-numbered priority value wins | string | `"1000"` | yes | + +## Requirements +### Installed Software + +- [Terraform](https://www.terraform.io/downloads.html) >= 1.3 +- [Terraform Provider for GCP](https://github.com/terraform-providers/terraform-provider-google) >= 4.25 +- [Terraform Provider for GCP Beta](https://github.com/terraform-providers/terraform-provider-google-beta) >= 4.25 +- [gcloud](https://cloud.google.com/sdk/gcloud/) >243.0.0 + +### Configure a Service Account +In order to execute this module you must have a Service Account with the following roles: + +- roles/compute.networkAdmin on the organization or folder + +If you are going to manage a Shared VPC, you must have either: + +- roles/compute.xpnAdmin on the organization +- roles/compute.xpnAdmin on the folder (beta) + +### Enable API's +In order to operate with the Service Account you must activate the following API on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com + +## Contributing + +Refer to the [contribution guidelines](./CONTRIBUTING.md) for +information on contributing to this module. + +[terraform-0.13-upgrade]: https://www.terraform.io/upgrade-guides/0-13.html +[2.6.0]: https://registry.terraform.io/modules/terraform-google-modules/network/google/2.6.0 diff --git a/community/modules/network/rdma-vpc/vpc-submodule/main.tf b/community/modules/network/rdma-vpc/vpc-submodule/main.tf new file mode 100644 index 0000000000..48ae8e705d --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/main.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + VPC configuration + *****************************************/ +resource "google_compute_network" "network" { + provider = google-beta + name = var.network_name + auto_create_subnetworks = var.auto_create_subnetworks + routing_mode = var.routing_mode + project = var.project_id + description = var.description + delete_default_routes_on_create = var.delete_default_internet_gateway_routes + mtu = var.mtu + enable_ula_internal_ipv6 = var.enable_ipv6_ula + internal_ipv6_range = var.internal_ipv6_range + network_firewall_policy_enforcement_order = var.network_firewall_policy_enforcement_order + network_profile = var.network_profile +} + +/****************************************** + Shared VPC + *****************************************/ +resource "google_compute_shared_vpc_host_project" "shared_vpc_host" { + count = var.shared_vpc_host ? 1 : 0 + project = var.project_id + depends_on = [google_compute_network.network] +} + + +/****************************************** + Subnet configuration + *****************************************/ +module "subnets" { + source = "terraform-google-modules/network/google//modules/subnets" + version = "~> 9.3" + project_id = var.project_id + network_name = google_compute_network.network.name + subnets = var.subnets + secondary_ranges = var.secondary_ranges +} + +/****************************************** + Routes + *****************************************/ +module "routes" { + source = "terraform-google-modules/network/google//modules/routes" + version = "~> 9.3" + project_id = var.project_id + network_name = google_compute_network.network.name + routes = var.routes + module_depends_on = [module.subnets.subnets] +} + +/****************************************** + Firewall rules + *****************************************/ +locals { + rules = [ + for f in var.firewall_rules : { + name = f.name + direction = f.direction + disabled = lookup(f, "disabled", null) + priority = lookup(f, "priority", null) + description = lookup(f, "description", null) + ranges = lookup(f, "ranges", null) + source_tags = lookup(f, "source_tags", null) + source_service_accounts = lookup(f, "source_service_accounts", null) + target_tags = lookup(f, "target_tags", null) + target_service_accounts = lookup(f, "target_service_accounts", null) + allow = lookup(f, "allow", []) + deny = lookup(f, "deny", []) + log_config = lookup(f, "log_config", null) + } + ] +} + +module "firewall_rules" { + source = "terraform-google-modules/network/google//modules/firewall-rules" + version = "~> 9.3" + project_id = var.project_id + network_name = google_compute_network.network.name + rules = local.rules + ingress_rules = var.ingress_rules + egress_rules = var.egress_rules +} diff --git a/community/modules/network/rdma-vpc/vpc-submodule/metadata.yaml b/community/modules/network/rdma-vpc/vpc-submodule/metadata.yaml new file mode 100644 index 0000000000..dad26c8e20 --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/metadata.yaml @@ -0,0 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spec: + requirements: + services: + - compute.googleapis.com diff --git a/community/modules/network/rdma-vpc/vpc-submodule/outputs.tf b/community/modules/network/rdma-vpc/vpc-submodule/outputs.tf new file mode 100644 index 0000000000..822bbcbdec --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/outputs.tf @@ -0,0 +1,90 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "network" { + value = google_compute_network.network + description = "The created network" +} + +output "subnets" { + value = module.subnets.subnets + description = "A map with keys of form subnet_region/subnet_name and values being the outputs of the google_compute_subnetwork resources used to create corresponding subnets." +} + +output "network_name" { + value = google_compute_network.network.name + description = "The name of the VPC being created" +} + +output "network_id" { + value = google_compute_network.network.id + description = "The ID of the VPC being created" +} + +output "network_self_link" { + value = google_compute_network.network.self_link + description = "The URI of the VPC being created" +} + +output "project_id" { + value = google_compute_network.network.project + description = "VPC project id" +} + +output "subnets_names" { + value = [for network in module.subnets.subnets : network.name] + description = "The names of the subnets being created" +} + +output "subnets_ids" { + value = [for network in module.subnets.subnets : network.id] + description = "The IDs of the subnets being created" +} + +output "subnets_ips" { + value = [for network in module.subnets.subnets : network.ip_cidr_range] + description = "The IPs and CIDRs of the subnets being created" +} + +output "subnets_self_links" { + value = [for network in module.subnets.subnets : network.self_link] + description = "The self-links of subnets being created" +} + +output "subnets_regions" { + value = [for network in module.subnets.subnets : network.region] + description = "The region where the subnets will be created" +} + +output "subnets_private_access" { + value = [for network in module.subnets.subnets : network.private_ip_google_access] + description = "Whether the subnets will have access to Google API's without a public IP" +} + +output "subnets_flow_logs" { + value = [for network in module.subnets.subnets : length(network.log_config) != 0 ? true : false] + description = "Whether the subnets will have VPC flow logs enabled" +} + +output "subnets_secondary_ranges" { + value = [for network in module.subnets.subnets : network.secondary_ip_range] + description = "The secondary ranges associated with these subnets" +} + +output "route_names" { + value = [for route in module.routes.routes : route.name] + description = "The route names associated with this VPC" +} diff --git a/community/modules/network/rdma-vpc/vpc-submodule/variables.tf b/community/modules/network/rdma-vpc/vpc-submodule/variables.tf new file mode 100644 index 0000000000..cf767f2fe8 --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/variables.tf @@ -0,0 +1,208 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project where this VPC will be created" + type = string +} + +variable "network_name" { + description = "The name of the network being created" + type = string +} + +variable "routing_mode" { + type = string + default = "GLOBAL" + description = "The network routing mode (default 'GLOBAL')" +} + +variable "shared_vpc_host" { + type = bool + description = "Makes this project a Shared VPC host if 'true' (default 'false')" + default = false +} + +variable "subnets" { + type = list(object({ + subnet_name = string + subnet_ip = string + subnet_region = string + subnet_private_access = optional(string) + subnet_private_ipv6_access = optional(string) + subnet_flow_logs = optional(string) + subnet_flow_logs_interval = optional(string) + subnet_flow_logs_sampling = optional(string) + subnet_flow_logs_metadata = optional(string) + subnet_flow_logs_filter = optional(string) + subnet_flow_logs_metadata_fields = optional(list(string)) + description = optional(string) + purpose = optional(string) + role = optional(string) + stack_type = optional(string) + ipv6_access_type = optional(string) + })) + description = "The list of subnets being created" +} + +variable "secondary_ranges" { + type = map(list(object({ range_name = string, ip_cidr_range = string }))) + description = "Secondary ranges that will be used in some of the subnets" + default = {} +} + +variable "routes" { + type = list(map(string)) + description = "List of routes being created in this VPC" + default = [] +} + +variable "firewall_rules" { + type = list(object({ + name = string + description = optional(string, null) + direction = optional(string, "INGRESS") + disabled = optional(bool, null) + priority = optional(number, null) + ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) + description = "This is DEPRECATED and available for backward compatibility. Use ingress_rules and egress_rules variables. List of firewall rules" + default = [] +} + +variable "delete_default_internet_gateway_routes" { + type = bool + description = "If set, ensure that all routes within the network specified whose names begin with 'default-route' and with a next hop of 'default-internet-gateway' are deleted" + default = false +} + + +variable "description" { + type = string + description = "An optional description of this resource. The resource must be recreated to modify this field." + default = "" +} + +variable "auto_create_subnetworks" { + type = bool + description = "When set to true, the network is created in 'auto subnet mode' and it will create a subnet for each region automatically across the 10.128.0.0/9 address range. When set to false, the network is created in 'custom subnet mode' so the user can explicitly connect subnetwork resources." + default = false +} + +variable "mtu" { + type = number + description = "The network MTU (If set to 0, meaning MTU is unset - defaults to '1460'). Recommended values: 1460 (default for historic reasons), 1500 (Internet default), or 8896 (for Jumbo packets). Allowed are all values in the range 1300 to 8896, inclusively." + default = 0 +} + +variable "ingress_rules" { + description = "List of ingress rules. This will be ignored if variable 'rules' is non-empty" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "egress_rules" { + description = "List of egress rules. This will be ignored if variable 'rules' is non-empty" + default = [] + type = list(object({ + name = string + description = optional(string, null) + disabled = optional(bool, null) + priority = optional(number, null) + destination_ranges = optional(list(string), []) + source_ranges = optional(list(string), []) + source_tags = optional(list(string)) + source_service_accounts = optional(list(string)) + target_tags = optional(list(string)) + target_service_accounts = optional(list(string)) + + allow = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + deny = optional(list(object({ + protocol = string + ports = optional(list(string)) + })), []) + log_config = optional(object({ + metadata = string + })) + })) +} + +variable "enable_ipv6_ula" { + type = bool + description = "Enabled IPv6 ULA, this is a permanent change and cannot be undone! (default 'false')" + default = false +} + +variable "internal_ipv6_range" { + type = string + default = null + description = "When enabling IPv6 ULA, optionally, specify a /48 from fd20::/20 (default null)" +} + +variable "network_firewall_policy_enforcement_order" { + type = string + default = null + description = "Set the order that Firewall Rules and Firewall Policies are evaluated. Valid values are `BEFORE_CLASSIC_FIREWALL` and `AFTER_CLASSIC_FIREWALL`. (default null or equivalent to `AFTER_CLASSIC_FIREWALL`)" +} + +variable "network_profile" { + # TODO Update this description + description = "Profile name for VPC configuration" + type = string + default = null +} diff --git a/community/modules/network/rdma-vpc/vpc-submodule/versions.tf b/community/modules/network/rdma-vpc/vpc-submodule/versions.tf new file mode 100644 index 0000000000..b75f5c66ee --- /dev/null +++ b/community/modules/network/rdma-vpc/vpc-submodule/versions.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.3" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 4.64" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 6.13.0" + } + } + + provider_meta "google" { + module_name = "blueprints/terraform/hpc-toolkit:rdma-vpc/experimental" + } +} diff --git a/community/modules/remote-desktop/chrome-remote-desktop/README.md b/community/modules/remote-desktop/chrome-remote-desktop/README.md index 85a942a045..5435827527 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/README.md +++ b/community/modules/remote-desktop/chrome-remote-desktop/README.md @@ -90,7 +90,7 @@ No resources. | [machine\_type](#input\_machine\_type) | Machine type to use for the instance creation. Must be N1 family if GPU is used. | `string` | `"n1-standard-8"` | no | | [metadata](#input\_metadata) | Metadata, provided as a map | `map(string)` | `{}` | no | | [name\_prefix](#input\_name\_prefix) | An optional name for all VM and disk resources.
If not supplied, `deployment_name` will be used.
When `name_prefix` is supplied, and `add_deployment_name_before_prefix` is set,
then resources are named by "<`deployment_name`>-<`name_prefix`>-<#>". | `string` | `null` | no | -| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | +| [network\_interfaces](#input\_network\_interfaces) | A list of network interfaces. The options match that of the terraform
network\_interface block of google\_compute\_instance. For descriptions of the
subfields or more information see the documentation:
https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance#nested_network_interface
**\_NOTE:\_** If `network_interfaces` are set, `network_self_link` and
`subnetwork_self_link` will be ignored, even if they are provided through
the `use` field. `bandwidth_tier` and `enable_public_ips` also do not apply
to network interfaces defined in this variable.
Subfields:
network (string, required if subnetwork is not supplied)
subnetwork (string, required if network is not supplied)
subnetwork\_project (string, optional)
network\_ip (string, optional)
nic\_type (string, optional, choose from ["GVNIC", "VIRTIO\_NET", "RDMA", "IRDMA", "MRDMA"])
stack\_type (string, optional, choose from ["IPV4\_ONLY", "IPV4\_IPV6"])
queue\_count (number, optional)
access\_config (object, optional)
ipv6\_access\_config (object, optional)
alias\_ip\_range (list(object), optional) | 
list(object({
    network            = string,
    subnetwork         = string,
    subnetwork_project = string,
    network_ip         = string,
    nic_type           = string,
    stack_type         = string,
    queue_count        = number,
    access_config = list(object({
      nat_ip                 = string,
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    ipv6_access_config = list(object({
      public_ptr_domain_name = string,
      network_tier           = string
    })),
    alias_ip_range = list(object({
      ip_cidr_range         = string,
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | | [network\_self\_link](#input\_network\_self\_link) | The self link of the network to attach the VM. | `string` | `"default"` | no | | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `"TERMINATE"` | no | diff --git a/community/modules/remote-desktop/chrome-remote-desktop/variables.tf b/community/modules/remote-desktop/chrome-remote-desktop/variables.tf index df31cf5f34..186275b80f 100644 --- a/community/modules/remote-desktop/chrome-remote-desktop/variables.tf +++ b/community/modules/remote-desktop/chrome-remote-desktop/variables.tf @@ -172,7 +172,7 @@ variable "network_interfaces" { subnetwork (string, required if network is not supplied) subnetwork_project (string, optional) network_ip (string, optional) - nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET"]) + nic_type (string, optional, choose from ["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"]) stack_type (string, optional, choose from ["IPV4_ONLY", "IPV4_IPV6"]) queue_count (number, optional) access_config (object, optional) diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/gpu_definition.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/gpu_definition.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-controller/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/gpu_definition.tf b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v5-login/gpu_definition.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v5-login/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md index d4ffeba6ed..be4e2dd5cd 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/README.md @@ -261,11 +261,11 @@ limitations under the License. | [daos\_network\_storage\_scripts](#module\_daos\_network\_storage\_scripts) | ../../../../modules/scripts/startup-script | n/a | | [nodeset\_cleanup](#module\_nodeset\_cleanup) | ./modules/cleanup_compute | n/a | | [nodeset\_cleanup\_tpu](#module\_nodeset\_cleanup\_tpu) | ./modules/cleanup_tpu | n/a | -| [slurm\_controller\_template](#module\_slurm\_controller\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 6.8.5 | +| [slurm\_controller\_template](#module\_slurm\_controller\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | b0575ab | | [slurm\_files](#module\_slurm\_files) | ./modules/slurm_files | n/a | | [slurm\_login\_instance](#module\_slurm\_login\_instance) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/_slurm_instance | 6.8.5 | -| [slurm\_login\_template](#module\_slurm\_login\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 6.8.5 | -| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | 6.8.5 | +| [slurm\_login\_template](#module\_slurm\_login\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | b0575ab | +| [slurm\_nodeset\_template](#module\_slurm\_nodeset\_template) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template | b0575ab | | [slurm\_nodeset\_tpu](#module\_slurm\_nodeset\_tpu) | github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_nodeset_tpu | 6.8.5 | ## Resources diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf index 4b455ed5bd..7536afdc06 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/controller.tf @@ -43,7 +43,7 @@ locals { # INSTANCE TEMPLATE module "slurm_controller_template" { - source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=6.8.5" + source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=b0575ab" project_id = var.project_id region = var.region diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/gpu_definition.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/gpu_definition.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf index dd8e4699ec..4bea11eec3 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/login.tf @@ -14,7 +14,7 @@ # TEMPLATE module "slurm_login_template" { - source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=6.8.5" + source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=b0575ab" for_each = { for x in var.login_nodes : x.name_prefix => x } diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf index 71a44a7236..dcb9698341 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-controller/partition.tf @@ -26,7 +26,7 @@ locals { # NODESET # TODO: remove dependency on slurm-gcp repo, move to local template module module "slurm_nodeset_template" { - source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=6.8.5" + source = "github.com/GoogleCloudPlatform/slurm-gcp.git//terraform/slurm_cluster/modules/slurm_instance_template?ref=b0575ab" for_each = local.nodeset_map project_id = var.project_id diff --git a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/gpu_definition.tf b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/community/modules/scheduler/schedmd-slurm-gcp-v6-login/gpu_definition.tf +++ b/community/modules/scheduler/schedmd-slurm-gcp-v6-login/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/examples/gke-a3-ultragpu.yaml b/examples/gke-a3-ultragpu.yaml new file mode 100644 index 0000000000..f25bfe6ccb --- /dev/null +++ b/examples/gke-a3-ultragpu.yaml @@ -0,0 +1,202 @@ +# Copyright 2024 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +blueprint_name: a3ultra-gke-demo + +vars: + project_id: gsc-demo-440819 + deployment_name: a3ultra-gke-demo + region: europe-west1 + zone: europe-west1-b + # Cidr block containing the IP of the machine calling terraform. + # The following line must be updated for this example to work. + + # For staging purposes authorized_cidr has been left completely open + # The value can be more specific if the IPs are known which will run kubectl + # e.g. the local system running Terraform or a remote node + authorized_cidr: 0.0.0.0/0 + extended_reservation: exr-gsc-5/reservationBlocks/exr-gsc-5-block-0001 + nccl_installer_path: /home/user/nccl_installer.yaml # please refer to README.md for where to find this file + +terraform_providers: + google: + source: hashicorp/google + version: 5.38.0 + configuration: + project: $(vars.project_id) + region: $(vars.region) + zone: $(vars.zone) + + google-beta: + source: hashicorp/google-beta + version: 5.38.0 + configuration: + project: $(vars.project_id) + region: $(vars.region) + zone: $(vars.zone) + + google-private: + source: hashicorp/google-private + version: 0.0.1962 # This version should not change - google-private is inherently brittle + configuration: + project: $(vars.project_id) + region: $(vars.region) + zone: $(vars.zone) + +deployment_groups: +- group: primary + modules: + - id: a3ultra-gke-net-0 + source: modules/network/vpc + settings: + network_name: a3ultra-gke-demo-net-0 + allowed_ssh_ip_ranges: [0.0.0.0/0] + subnetworks: + - subnet_name: a3ultra-gke-demo-sub-0 + subnet_region: $(vars.region) + subnet_ip: 192.168.0.0/18 + secondary_ranges: + a3ultra-gke-demo-sub-0: + - range_name: pods + ip_cidr_range: 10.4.0.0/14 + - range_name: services + ip_cidr_range: 10.0.32.0/20 + firewall_rules: + - name: a3ultra-gke-demo-internal-0 + ranges: [192.168.0.0/16] + allow: + - protocol: tcp + ports: ["0-65535"] + - protocol: udp + ports: ["0-65535"] + - protocol: icmp + + - id: a3ultra-gke-net-1 + source: modules/network/vpc + settings: + network_name: a3ultra-gke-demo-net-1 + subnetworks: + - subnet_name: a3ultra-gke-demo-sub-1 + subnet_region: $(vars.region) + subnet_ip: 192.168.64.0/18 + firewall_rules: + - name: a3ultra-gke-demo-internal-1 + ranges: [192.168.0.0/16] + allow: + - protocol: tcp + ports: ["0-65535"] + - protocol: udp + ports: ["0-65535"] + - protocol: icmp + + - id: a3ultra-gke-rdma-net + source: community/modules/network/rdma-vpc + settings: + network_name: a3ultra-gke-demo-rdma-net + network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-roce + network_routing_mode: REGIONAL + subnetworks_template: + name_prefix: a3ultra-gke-demo-rdma-sub + count: 8 + ip_range: 192.168.128.0/18 + region: $(vars.region) + + - id: a3-ultragpu-cluster + source: modules/scheduler/gke-cluster + use: [a3ultra-gke-net-0] + settings: + cluster_availability_type: MULTI_ZONAL + cluster_reference_type: NAME + system_node_pool_machine_type: "e2-standard-16" + enable_private_endpoint: false # Allows access from authorized public IPs + zone: $(vars.zone) # Temporarily used to target the GKE sandbox + master_authorized_networks: + - cidr_block: $(vars.authorized_cidr) # Allows your machine run kubectl command. It's required for the multi-network setup. + display_name: "kubectl-access-network" + additional_networks: + $(concat( + [{ + network=a3ultra-gke-net-1.network_name, + subnetwork=a3ultra-gke-net-1.subnetwork_name, + subnetwork_project=vars.project_id, + nic_type="GVNIC", + queue_count=null, + network_ip=null, + stack_type=null, + access_config=[{nat_ip=null, public_ptr_domain_name=null, network_tier=null}], + ipv6_access_config=[], + alias_ip_range=[] + }], + a3ultra-gke-rdma-net.subnetwork_interfaces_gke + )) + outputs: [instructions] + + - id: data-bucket + source: community/modules/file-system/cloud-storage-bucket + settings: + local_mount: /data + random_suffix: true + force_destroy: true + + - id: data-bucket-pv + source: modules/file-system/gke-persistent-volume + use: [a3-ultragpu-cluster, data-bucket] + settings: {capacity_gb: 50} + + - id: a3-ultragpu-pool + source: modules/compute/gke-node-pool + use: [a3-ultragpu-cluster] + settings: + machine_type: a3-ultragpu-8g + zones: [$(vars.zone)] + disk_type: hyperdisk-balanced + static_node_count: 1 + guest_accelerator: + - type: nvidia-h200-141gb + count: 8 + gpu_driver_installation_config: + - gpu_driver_version: "LATEST" + local_ssd_count_nvme_block: 32 + reservation_affinity: + consume_reservation_type: SPECIFIC_RESERVATION + specific_reservations: + - name: $(vars.extended_reservation) + outputs: [instructions] + + - id: workload_manager_install + source: modules/management/kubectl-apply + use: [a3-ultragpu-cluster] + settings: + kueue: + install: true + jobset: + install: true + apply_manifests: + - source: $(vars.nccl_installer_path) + + - id: job_template + source: modules/compute/gke-job-template + use: [a3-ultragpu-pool] + settings: + image: nvidia/cuda:11.0.3-runtime-ubuntu20.04 + command: + - nvidia-smi + node_count: 2 + name: run-nvidia-smi + outputs: [instructions] + +terraform_backend_defaults: + type: gcs + configuration: + bucket: terraform-state-gsc-demo-440819 diff --git a/modules/compute/gke-node-pool/README.md b/modules/compute/gke-node-pool/README.md index 7359e934ca..e80b60f865 100644 --- a/modules/compute/gke-node-pool/README.md +++ b/modules/compute/gke-node-pool/README.md @@ -333,6 +333,7 @@ limitations under the License. | [local\_ssd\_count\_nvme\_block](#input\_local\_ssd\_count\_nvme\_block) | The number of local SSDs to attach to each node to back block storage.
Uses NVMe interfaces. Must be supported by `machine_type`.
When set to null, default value either is [set based on machine\_type](https://cloud.google.com/compute/docs/disks/local-ssd#choose_number_local_ssds) or GKE decides about default value.
[See above](#local-ssd-storage) for more info. | `number` | `null` | no | | [machine\_type](#input\_machine\_type) | The name of a Google Compute Engine machine type. | `string` | `"c2-standard-60"` | no | | [name](#input\_name) | The name of the node pool. If not set, automatically populated by machine type and module id (unique blueprint-wide) as suffix.
If setting manually, ensure a unique value across all gke-node-pools. | `string` | `null` | no | +| [node\_version](#input\_node\_version) | Temporary variable to explicitly set the node version | `string` | `null` | no | | [placement\_policy](#input\_placement\_policy) | Group placement policy to use for the node pool's nodes. `COMPACT` is the only supported value for `type` currently. `name` is the name of the placement policy.
It is assumed that the specified policy exists. To create a placement policy refer to https://cloud.google.com/sdk/gcloud/reference/compute/resource-policies/create/group-placement.
Note: Placement policies have the [following](https://cloud.google.com/compute/docs/instances/placement-policies-overview#restrictions-compact-policies) restrictions. | 
object({
    type = string
    name = optional(string)
  })
| 
{
  "name": null,
  "type": null
}
| no | | [project\_id](#input\_project\_id) | The project ID to host the cluster in. | `string` | n/a | yes | | [reservation\_affinity](#input\_reservation\_affinity) | Reservation resource to consume. When targeting SPECIFIC\_RESERVATION, specific\_reservations needs be specified.
Even though specific\_reservations is a list, only one reservation is allowed by the NodePool API.
It is assumed that the specified reservation exists and has available capacity.
For a shared reservation, specify the project\_id as well in which it was created.
To create a reservation refer to https://cloud.google.com/compute/docs/instances/reservations-single-project and https://cloud.google.com/compute/docs/instances/reservations-shared | 
object({
    consume_reservation_type = string
    specific_reservations = optional(list(object({
      name    = string
      project = optional(string)
    })))
  })
| 
{
  "consume_reservation_type": "NO_RESERVATION",
  "specific_reservations": []
}
| no | diff --git a/modules/compute/gke-node-pool/disk_definitions.tf b/modules/compute/gke-node-pool/disk_definitions.tf index f7dbebea0a..3afefa9354 100644 --- a/modules/compute/gke-node-pool/disk_definitions.tf +++ b/modules/compute/gke-node-pool/disk_definitions.tf @@ -22,8 +22,9 @@ locals { local_ssd_machines = { - "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, - "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = 16, local_ssd_count_nvme_block = null }, + "a3-highgpu-8g" = { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = 16 }, + "a3-megagpu-8g" = { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = 16 }, + "a3-ultragpu-8g" = { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = 32 }, } generated_local_ssd_config = lookup(local.local_ssd_machines, var.machine_type, { local_ssd_count_ephemeral_storage = null, local_ssd_count_nvme_block = null }) diff --git a/modules/compute/gke-node-pool/gpu_definition.tf b/modules/compute/gke-node-pool/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/modules/compute/gke-node-pool/gpu_definition.tf +++ b/modules/compute/gke-node-pool/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/modules/compute/gke-node-pool/main.tf b/modules/compute/gke-node-pool/main.tf index 3e38564988..b481d0254f 100644 --- a/modules/compute/gke-node-pool/main.tf +++ b/modules/compute/gke-node-pool/main.tf @@ -40,6 +40,7 @@ resource "google_container_node_pool" "node_pool" { name = coalesce(var.name, "${var.machine_type}-${local.module_unique_id}") cluster = var.cluster_id node_locations = var.zones + version = var.node_version node_count = var.static_node_count dynamic "autoscaling" { @@ -177,7 +178,10 @@ resource "google_container_node_pool" "node_pool" { reservation_affinity { consume_reservation_type = var.reservation_affinity.consume_reservation_type key = length(local.verified_specific_reservations) != 1 ? null : local.reservation_resource_api_label - values = length(local.verified_specific_reservations) != 1 ? null : [for r in local.verified_specific_reservations : "projects/${r.project}/reservations/${r.name}"] + values = length(local.verified_specific_reservations) != 1 ? null : [ + for i, r in local.verified_specific_reservations : + (length(local.input_reservation_suffixes[i]) > 0 ? format("%s%s", r.name, local.input_reservation_suffixes[i]) : "projects/${r.project}/reservations/${r.name}") + ] } dynamic "host_maintenance_policy" { @@ -250,6 +254,14 @@ resource "google_container_node_pool" "node_pool" { %{endfor} EOT } + precondition { + condition = ( + (local.input_specific_reservations_count == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) == 0) || + (local.input_specific_reservations_count == 1 && length(local.input_reservation_suffixes) > 0 && try(local.input_reservation_projects[0], var.project_id) == var.project_id) + ) + error_message = "Shared Extended reservations are not supported by GKE." + } } } diff --git a/modules/compute/gke-node-pool/reservation_definitions.tf b/modules/compute/gke-node-pool/reservation_definitions.tf index 26ab22808f..bf6f2639b7 100644 --- a/modules/compute/gke-node-pool/reservation_definitions.tf +++ b/modules/compute/gke-node-pool/reservation_definitions.tf @@ -14,6 +14,16 @@ * limitations under the License. */ +# Split the input into three different lists where the details of a given reservation are at the same index across these lists. +locals { + # Specific block of an extended reservation can be targeted with exr-one/reservationBlocks/exr-one-block-1 + # Data source needs to be queried with the reservation name only. So, we extract the reservation name + input_reservation_names = [for r in try(var.reservation_affinity.specific_reservations, []) : split("/", r.name)[0]] + input_reservation_projects = [for r in try(var.reservation_affinity.specific_reservations, []) : coalesce(r.project, var.project_id)] + # We, also, remember the suffix "/reservationBlocks/exr-one-block-1" for use elsewhere afterwards + input_reservation_suffixes = [for r in try(var.reservation_affinity.specific_reservations, []) : substr(r.name, length(split("/", r.name)[0]), -1)] +} + data "google_compute_reservation" "specific_reservations" { for_each = ( local.input_specific_reservations_count == 0 ? @@ -21,11 +31,11 @@ data "google_compute_reservation" "specific_reservations" { { for pair in flatten([ for zone in try(var.zones, []) : [ - for reservation in try(var.reservation_affinity.specific_reservations, []) : { - key : "${coalesce(reservation.project, var.project_id)}/${zone}/${reservation.name}" + for i, reservation_name in try(local.input_reservation_names, []) : { + key : "${local.input_reservation_projects[i]}/${zone}/${reservation_name}" zone : zone - reservation_name : reservation.name - project : reservation.project == null ? var.project_id : reservation.project + reservation_name : reservation_name + project : local.input_reservation_projects[i] } ] ]) : diff --git a/modules/compute/gke-node-pool/variables.tf b/modules/compute/gke-node-pool/variables.tf index eecc4634c1..d7e0e40901 100644 --- a/modules/compute/gke-node-pool/variables.tf +++ b/modules/compute/gke-node-pool/variables.tf @@ -385,3 +385,9 @@ variable "gke_version" { description = "GKE version" type = string } + +variable "node_version" { + description = "Temporary variable to explicitly set the node version" + type = string + default = null +} diff --git a/modules/compute/vm-instance/README.md b/modules/compute/vm-instance/README.md index 149f472d68..7c65d472e6 100644 --- a/modules/compute/vm-instance/README.md +++ b/modules/compute/vm-instance/README.md @@ -171,6 +171,7 @@ limitations under the License. | [terraform](#requirement\_terraform) | >= 1.3.0 | | [google](#requirement\_google) | >= 4.73.0 | | [google-beta](#requirement\_google-beta) | >= 4.73.0 | +| [google-private](#requirement\_google-private) | >= 0.0.1962 | | [null](#requirement\_null) | >= 3.0 | ## Providers @@ -179,6 +180,7 @@ limitations under the License. |------|---------| | [google](#provider\_google) | >= 4.73.0 | | [google-beta](#provider\_google-beta) | >= 4.73.0 | +| [google-private](#provider\_google-private) | >= 0.0.1962 | | [null](#provider\_null) | >= 3.0 | ## Modules @@ -191,12 +193,13 @@ limitations under the License. | Name | Type | |------|------| -| [google-beta_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_instance) | resource | | [google-beta_google_compute_resource_policy.placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/google_compute_resource_policy) | resource | +| [google-private_google_compute_instance.compute_vm](https://registry.terraform.io/providers/hashicorp/google-private/latest/docs/resources/google_compute_instance) | resource | | [google_compute_address.compute_ip](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_address) | resource | | [google_compute_disk.boot_disk](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_disk) | resource | | [null_resource.image](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.replace_vm_trigger_from_placement](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [google-beta_google_compute_resource_policy.pre_existing_placement_policy](https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/data-sources/google_compute_resource_policy) | data source | | [google_compute_image.compute_image](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source | ## Inputs @@ -229,6 +232,7 @@ limitations under the License. | [network\_storage](#input\_network\_storage) | An array of network attached storage mounts to be configured. | 
list(object({
    server_ip             = string,
    remote_mount          = string,
    local_mount           = string,
    fs_type               = string,
    mount_options         = string,
    client_install_runner = map(string)
    mount_runner          = map(string)
  }))
| `[]` | no | | [on\_host\_maintenance](#input\_on\_host\_maintenance) | Describes maintenance behavior for the instance. If left blank this will default to `MIGRATE` except for when `placement_policy`, spot provisioning, or GPUs require it to be `TERMINATE` | `string` | `null` | no | | [placement\_policy](#input\_placement\_policy) | Control where your VM instances are physically located relative to each other within a zone.
See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_resource_policy#nested_group_placement_policy | `any` | `null` | no | +| [pre\_existing\_placement\_policy](#input\_pre\_existing\_placement\_policy) | Use pre-existing placement policy within the project specified | `string` | `null` | no | | [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created | `string` | n/a | yes | | [region](#input\_region) | The region to deploy to | `string` | n/a | yes | | [service\_account](#input\_service\_account) | DEPRECATED - Use `service_account_email` and `service_account_scopes` instead. | 
object({
    email  = string,
    scopes = set(string)
  })
| `null` | no | diff --git a/modules/compute/vm-instance/gpu_definition.tf b/modules/compute/vm-instance/gpu_definition.tf index 6c5d96d286..1c84a92721 100644 --- a/modules/compute/vm-instance/gpu_definition.tf +++ b/modules/compute/vm-instance/gpu_definition.tf @@ -38,6 +38,7 @@ locals { "a2-ultragpu-8g" = { type = "nvidia-a100-80gb", count = 8 }, "a3-highgpu-8g" = { type = "nvidia-h100-80gb", count = 8 }, "a3-megagpu-8g" = { type = "nvidia-h100-mega-80gb", count = 8 }, + "a3-ultragpu-8g" = { type = "nvidia-h200-141gb", count = 8 }, "g2-standard-4" = { type = "nvidia-l4", count = 1 }, "g2-standard-8" = { type = "nvidia-l4", count = 1 }, "g2-standard-12" = { type = "nvidia-l4", count = 1 }, diff --git a/modules/compute/vm-instance/main.tf b/modules/compute/vm-instance/main.tf index c639f075d6..748ccc578f 100644 --- a/modules/compute/vm-instance/main.tf +++ b/modules/compute/vm-instance/main.tf @@ -123,6 +123,15 @@ resource "google_compute_disk" "boot_disk" { } } +data "google_compute_resource_policy" "pre_existing_placement_policy" { + project = var.project_id + provider = google-beta + + count = var.pre_existing_placement_policy != null ? 1 : 0 + name = var.pre_existing_placement_policy + region = var.region +} + resource "google_compute_resource_policy" "placement_policy" { project = var.project_id provider = google-beta @@ -165,7 +174,7 @@ resource "google_compute_address" "compute_ip" { resource "google_compute_instance" "compute_vm" { project = var.project_id - provider = google-beta + provider = google-private count = var.instance_count @@ -176,7 +185,8 @@ resource "google_compute_instance" "compute_vm" { machine_type = var.machine_type zone = var.zone - resource_policies = google_compute_resource_policy.placement_policy[*].self_link + resource_policies = coalesce(google_compute_resource_policy.placement_policy[*].self_link, + data.google_compute_resource_policy.pre_existing_placement_policy[*].self_link) tags = var.tags labels = local.labels @@ -297,5 +307,10 @@ resource "google_compute_instance" "compute_vm" { ], "${substr(var.machine_type, 0, 3)}:${var.disk_type}") error_message = "A disk_type=${var.disk_type} cannot be used with machine_type=${var.machine_type}." } + precondition { + condition = (length(google_compute_resource_policy.placement_policy) == 0 || + length(data.google_compute_resource_policy.pre_existing_placement_policy) == 0) + error_message = "Pre-existing placement policy and placement policy variables are mutually exclusive" + } } } diff --git a/modules/compute/vm-instance/variables.tf b/modules/compute/vm-instance/variables.tf index a874ddf825..6eca689c07 100644 --- a/modules/compute/vm-instance/variables.tf +++ b/modules/compute/vm-instance/variables.tf @@ -223,9 +223,9 @@ variable "network_interfaces" { } validation { condition = alltrue([ - for ni in var.network_interfaces : ni.nic_type == "GVNIC" || ni.nic_type == "VIRTIO_NET" || ni.nic_type == null + for ni in var.network_interfaces : contains(["GVNIC", "VIRTIO_NET", "RDMA", "IRDMA", "MRDMA"], coalesce(ni.nic_type, "INVALID")) || ni.nic_type == null ]) - error_message = "In the variable network_interfaces, field \"nic_type\" must be either \"GVNIC\", \"VIRTIO_NET\" or null." + error_message = "In the variable network_interfaces, field \"nic_type\" must be either \"GVNIC\", \"VIRTIO_NET\", \"RDMA\", \"IRDMA\", \"MRDMA\", or null." } validation { condition = alltrue([ @@ -327,6 +327,12 @@ variable "placement_policy" { } } +variable "pre_existing_placement_policy" { + description = "Use pre-existing placement policy within the project specified" + type = string + default = null +} + variable "spot" { description = "Provision VMs using discounted Spot pricing, allowing for preemption" type = bool diff --git a/modules/compute/vm-instance/versions.tf b/modules/compute/vm-instance/versions.tf index c6577b1edb..27d872e995 100644 --- a/modules/compute/vm-instance/versions.tf +++ b/modules/compute/vm-instance/versions.tf @@ -25,6 +25,12 @@ terraform { source = "hashicorp/google-beta" version = ">= 4.73.0" } + + google-private = { + source = "hashicorp/google-private" + version = ">= 0.0.1962" + } + null = { source = "hashicorp/null" version = ">= 3.0" @@ -36,6 +42,9 @@ terraform { provider_meta "google-beta" { module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.41.0" } + provider_meta "google-private" { + module_name = "blueprints/terraform/hpc-toolkit:vm-instance/v1.38.0" + } required_version = ">= 1.3.0" } diff --git a/modules/scheduler/gke-cluster/README.md b/modules/scheduler/gke-cluster/README.md index f8a5487563..c829056416 100644 --- a/modules/scheduler/gke-cluster/README.md +++ b/modules/scheduler/gke-cluster/README.md @@ -144,6 +144,8 @@ limitations under the License. | [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks enables multi networking and creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | | [authenticator\_security\_group](#input\_authenticator\_security\_group) | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | [autoscaling\_profile](#input\_autoscaling\_profile) | (Beta) Optimize for utilization or availability when deciding to remove nodes. Can be BALANCED or OPTIMIZE\_UTILIZATION. | `string` | `"OPTIMIZE_UTILIZATION"` | no | +| [cluster\_availability\_type](#input\_cluster\_availability\_type) | Type of cluster availability. Possible values are: {REGIONAL, MULTI\_ZONAL} | `string` | `"REGIONAL"` | no | +| [cluster\_reference\_type](#input\_cluster\_reference\_type) | How the google\_container\_node\_pool.system\_node\_pools refers to the cluster. Possible values are: {SELF\_LINK, NAME} | `string` | `"SELF_LINK"` | no | | [configure\_workload\_identity\_sa](#input\_configure\_workload\_identity\_sa) | When true, a kubernetes service account will be created and bound using workload identity to the service account used to create the cluster. | `bool` | `false` | no | | [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment. Used in the GKE cluster name by default and can be configured with `prefix_with_deployment_name`. | `string` | n/a | yes | | [enable\_dataplane\_v2](#input\_enable\_dataplane\_v2) | Enables [Dataplane v2](https://cloud.google.com/kubernetes-engine/docs/concepts/dataplane-v2). This setting is immutable on clusters. If null, will default to false unless using multi-networking, in which case it will default to true | `bool` | `null` | no | @@ -187,6 +189,7 @@ limitations under the License. | [system\_node\_pool\_taints](#input\_system\_node\_pool\_taints) | Taints to be applied to the system node pool. | 
list(object({
    key    = string
    value  = any
    effect = string
  }))
| 
[
  {
    "effect": "NO_SCHEDULE",
    "key": "components.gke.io/gke-managed-components",
    "value": true
  }
]
| no | | [timeout\_create](#input\_timeout\_create) | Timeout for creating a node pool | `string` | `null` | no | | [timeout\_update](#input\_timeout\_update) | Timeout for updating a node pool | `string` | `null` | no | +| [zone](#input\_zone) | Zone for a zonal cluster | `string` | `null` | no | ## Outputs diff --git a/modules/scheduler/gke-cluster/main.tf b/modules/scheduler/gke-cluster/main.tf index 79b1051d78..6ba678a258 100644 --- a/modules/scheduler/gke-cluster/main.tf +++ b/modules/scheduler/gke-cluster/main.tf @@ -48,7 +48,7 @@ resource "google_container_cluster" "gke_cluster" { project = var.project_id name = local.name - location = var.region + location = var.cluster_availability_type == "MULTI_ZONAL" ? var.zone : var.region resource_labels = local.labels # decouple node pool lifecycle from cluster life cycle @@ -185,6 +185,15 @@ resource "google_container_cluster" "gke_cluster" { condition = !(!coalesce(var.enable_multi_networking, true) && length(var.additional_networks) > 0) error_message = "'enable_multi_networking' cannot be false when using multivpc module, which passes additional_networks." } + precondition { + condition = contains(["REGIONAL", "MULTI_ZONAL"], var.cluster_availability_type) + error_message = "`cluster_availability_type` must be one of {REGIONAL, MULTI_ZONAL}" + } + precondition { + condition = contains(["SELF_LINK", "NAME"], var.cluster_reference_type) + error_message = "`cluster_reference_type` must be one of {SELF_LINK, NAME}" + } + } logging_service = "logging.googleapis.com/kubernetes" @@ -197,9 +206,12 @@ resource "google_container_node_pool" "system_node_pools" { provider = google-beta count = var.system_node_pool_enabled ? 1 : 0 - project = var.project_id - name = var.system_node_pool_name - cluster = google_container_cluster.gke_cluster.self_link + project = var.project_id + name = var.system_node_pool_name + cluster = var.cluster_reference_type == "NAME" ? google_container_cluster.gke_cluster.name : google_container_cluster.gke_cluster.self_link + version = var.min_master_version + location = var.cluster_availability_type == "MULTI_ZONAL" ? var.zone : null + autoscaling { total_min_node_count = var.system_node_pool_node_count.total_min_nodes total_max_node_count = var.system_node_pool_node_count.total_max_nodes @@ -318,14 +330,15 @@ module "kubectl_apply" { { source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", template_vars = { - name = "vpc${idx + 1}", + name = network_info.subnetwork, network_name = network_info.network - subnetwork_name = network_info.subnetwork + subnetwork_name = network_info.subnetwork, + device_mode = strcontains(upper(network_info.nic_type), "RDMA") ? "RDMA" : "NetDevice" } }, { source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "vpc${idx + 1}" } + template_vars = { name = network_info.subnetwork } } ] ]) diff --git a/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl b/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl index fb7f0dba83..d376a1a760 100644 --- a/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ b/modules/scheduler/gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -6,4 +6,4 @@ metadata: spec: vpc: ${network_name} vpcSubnet: ${subnetwork_name} - deviceMode: NetDevice + deviceMode: ${device_mode} diff --git a/modules/scheduler/gke-cluster/variables.tf b/modules/scheduler/gke-cluster/variables.tf index e0974fb126..a6dc363e47 100644 --- a/modules/scheduler/gke-cluster/variables.tf +++ b/modules/scheduler/gke-cluster/variables.tf @@ -345,3 +345,21 @@ variable "additional_networks" { })) })) } + +variable "cluster_reference_type" { + description = "How the google_container_node_pool.system_node_pools refers to the cluster. Possible values are: {SELF_LINK, NAME}" + default = "SELF_LINK" + type = string +} + +variable "cluster_availability_type" { + description = "Type of cluster availability. Possible values are: {REGIONAL, MULTI_ZONAL}" + default = "REGIONAL" + type = string +} + +variable "zone" { + description = "Zone for a zonal cluster" + default = null + type = string +} diff --git a/modules/scheduler/pre-existing-gke-cluster/README.md b/modules/scheduler/pre-existing-gke-cluster/README.md index 4caf7ff258..aaac5a547f 100644 --- a/modules/scheduler/pre-existing-gke-cluster/README.md +++ b/modules/scheduler/pre-existing-gke-cluster/README.md @@ -103,6 +103,7 @@ limitations under the License. | [additional\_networks](#input\_additional\_networks) | Additional network interface details for GKE, if any. Providing additional networks creates relevat network objects on the cluster. | 
list(object({
    network            = string
    subnetwork         = string
    subnetwork_project = string
    network_ip         = string
    nic_type           = string
    stack_type         = string
    queue_count        = number
    access_config = list(object({
      nat_ip       = string
      network_tier = string
    }))
    ipv6_access_config = list(object({
      network_tier = string
    }))
    alias_ip_range = list(object({
      ip_cidr_range         = string
      subnetwork_range_name = string
    }))
  }))
| `[]` | no | | [cluster\_name](#input\_cluster\_name) | Name of the existing cluster | `string` | n/a | yes | | [project\_id](#input\_project\_id) | Project that hosts the existing cluster | `string` | n/a | yes | +| [rdma\_subnetwork\_name\_prefix](#input\_rdma\_subnetwork\_name\_prefix) | Prefix of the RDMA subnetwork names | `string` | `null` | no | | [region](#input\_region) | Region in which to search for the cluster | `string` | n/a | yes | ## Outputs diff --git a/modules/scheduler/pre-existing-gke-cluster/main.tf b/modules/scheduler/pre-existing-gke-cluster/main.tf index 4b65ebe365..926d2be100 100644 --- a/modules/scheduler/pre-existing-gke-cluster/main.tf +++ b/modules/scheduler/pre-existing-gke-cluster/main.tf @@ -20,26 +20,51 @@ data "google_container_cluster" "existing_gke_cluster" { location = var.region } -module "kubectl_apply" { - source = "../../management/kubectl-apply" # can point to github - - cluster_id = data.google_container_cluster.existing_gke_cluster.id - project_id = var.project_id +locals { + rdma_networks = [for network_info in var.additional_networks : network_info if strcontains(upper(network_info.nic_type), "RDMA")] + non_rdma_networks = [for network_info in var.additional_networks : network_info if !strcontains(upper(network_info.nic_type), "RDMA")] + apply_manifests_rdma_networks = flatten([ + for idx, network_info in local.rdma_networks : [ + { + source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", + template_vars = { + name = "${var.rdma_subnetwork_name_prefix}-${idx}", + network_name = network_info.network + subnetwork_name = "${var.rdma_subnetwork_name_prefix}-${idx}", + device_mode = "RDMA" + } + }, + { + source = "${path.module}/templates/network-object.yaml.tftpl", + template_vars = { name = "${var.rdma_subnetwork_name_prefix}-${idx}" } + } + ] + ]) - apply_manifests = flatten([ - for idx, network_info in var.additional_networks : [ + apply_manifests_non_rdma_networks = flatten([ + for idx, network_info in local.non_rdma_networks : [ { source = "${path.module}/templates/gke-network-paramset.yaml.tftpl", template_vars = { - name = "vpc${idx + 1}", + name = network_info.subnetwork network_name = network_info.network subnetwork_name = network_info.subnetwork + device_mode = "NetDevice" } }, { source = "${path.module}/templates/network-object.yaml.tftpl", - template_vars = { name = "vpc${idx + 1}" } + template_vars = { name = network_info.subnetwork } } ] ]) } + +module "kubectl_apply" { + source = "../../management/kubectl-apply" + + cluster_id = data.google_container_cluster.existing_gke_cluster.id + project_id = var.project_id + + apply_manifests = concat(local.apply_manifests_non_rdma_networks, local.apply_manifests_rdma_networks) +} diff --git a/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl b/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl index fb7f0dba83..d376a1a760 100644 --- a/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl +++ b/modules/scheduler/pre-existing-gke-cluster/templates/gke-network-paramset.yaml.tftpl @@ -6,4 +6,4 @@ metadata: spec: vpc: ${network_name} vpcSubnet: ${subnetwork_name} - deviceMode: NetDevice + deviceMode: ${device_mode} diff --git a/modules/scheduler/pre-existing-gke-cluster/variables.tf b/modules/scheduler/pre-existing-gke-cluster/variables.tf index 67e7a24dca..9e9ed98ed3 100644 --- a/modules/scheduler/pre-existing-gke-cluster/variables.tf +++ b/modules/scheduler/pre-existing-gke-cluster/variables.tf @@ -53,3 +53,9 @@ variable "additional_networks" { })) })) } + +variable "rdma_subnetwork_name_prefix" { + description = "Prefix of the RDMA subnetwork names" + default = null + type = string +}