Merge pull request #1354 from GoogleCloudPlatform/release-candidate

Release v1.18.0

.ansible-lint

@@ -3,20 +3,20 @@ skip_list:
 - jinja[invalid]
 
 exclude_paths:
-  - .cache/ # implicit unless exclude_paths is defined in config
-  - .github/
-  - cmd/
-  - docs/
-  - examples/
-  - community/examples/
-  - pkg/
+- .cache/ # implicit unless exclude_paths is defined in config
+- .github/
+- cmd/
+- docs/
+- examples/
+- community/examples/
+- pkg/
 
 mock_roles:
 - googlecloudplatform.google_cloud_ops_agents
 
 kinds:
-  - playbook: "**/ansible_playbooks/*test.{yml,yaml}"
-  - playbook: "**/files/*.{yml,yaml}"
-  - playbook: "**/scripts/*.{yml,yaml}"
-  - tasks: "**/ansible_playbooks/test*.{yml,yaml}"
-  - tasks: "**/tasks/*"
+- playbook: "**/ansible_playbooks/*test.{yml,yaml}"
+- playbook: "**/files/*.{yml,yaml}"
+- playbook: "**/scripts/*.{yml,yaml}"
+- tasks: "**/ansible_playbooks/test*.{yml,yaml}"
+- tasks: "**/tasks/*"

.gitignore

@@ -59,5 +59,5 @@ packer-manifest.json
 *.auto.pkrvars.hcl
 
 #### Exclude from gitingore
-!tools/validate_configs/golden_copies/*/*/*/defaults.auto.pkrvars.hcl
-!tools/validate_configs/golden_copies/*/*/terraform.tfvars
+!tools/validate_configs/golden_copies/expectations/*/*/*/defaults.auto.pkrvars.hcl
+!tools/validate_configs/golden_copies/expectations/*/*/terraform.tfvars

.pre-commit-config.yaml

@@ -66,15 +66,6 @@ repos:
   hooks:
   - id: go-critic
     args: [-disable, "#experimental,sloppyTypeAssert"]
-- repo: https://github.com/ansible/ansible-lint
-  rev: v6.11.0
-  hooks:
-  - id: ansible-lint
-    always_run: false
-    exclude: '^(docs/|examples/|community/examples/)'
-    types: [yaml]
-    additional_dependencies:
-    - ansible==6.*
 - repo: https://github.com/adrienverge/yamllint
   rev: v1.29.0
   hooks:

Makefile

@@ -1,6 +1,6 @@
 # PREAMBLE
-MIN_PACKER_VERSION=1.6 # for building images
-MIN_TERRAFORM_VERSION=1.0 # for deploying modules
+MIN_PACKER_VERSION=1.7.9 # for building images
+MIN_TERRAFORM_VERSION=1.2 # for deploying modules
 MIN_GOLANG_VERSION=1.18 # for building ghpc
 
 .PHONY: install install-user tests format add-google-license install-dev-deps \
@@ -117,8 +117,9 @@ else
 ## GO IS PRESENT
 warn-go-missing:
 
-GO_VERSION_CHECK=$(shell expr `go version | cut -f 3 -d ' ' | cut -f 1 -d '-' | cut -c 3-` \>= $(MIN_GOLANG_VERSION))
-ifneq ("$(GO_VERSION_CHECK)", "1")
+GO_VERSION=$(shell go version | cut -f 3 -d ' ')
+GO_VERSION_CHECK=$(shell ./tools/version_check.sh $(GO_VERSION) $(MIN_GOLANG_VERSION))
+ifneq ("$(GO_VERSION_CHECK)", "yes")
 warn-go-version:
 	$(warning WARNING: Go version must be greater than $(MIN_GOLANG_VERSION), update at  https://go.dev/doc/install)
 else
@@ -145,8 +146,9 @@ else
 ## TERRAFORM IS PRESENT
 warn-terraform-missing:
 
-TF_VERSION_CHECK=$(shell expr `terraform version | cut -f 2- -d ' ' | cut -c 2- | head -n1` \>= $(MIN_TERRAFORM_VERSION))
-ifneq ("$(TF_VERSION_CHECK)", "1")
+TF_VERSION=$(shell terraform version | cut -f 2- -d ' ' | head -n1)
+TF_VERSION_CHECK=$(shell ./tools/version_check.sh $(TF_VERSION) $(MIN_TERRAFORM_VERSION))
+ifneq ("$(TF_VERSION_CHECK)", "yes")
 warn-terraform-version:
 	$(warning WARNING: terraform version must be greater than $(MIN_TERRAFORM_VERSION), update at https://learn.hashicorp.com/tutorials/terraform/install-cli)
 else
@@ -159,7 +161,7 @@ validate_configs: ghpc
 
 validate_golden_copy: ghpc
 	$(info *********** running "Golden copy" tests ***********)
-	tools/validate_configs/validate_golden_copy.sh
+	tools/validate_configs/golden_copies/validate.sh
 
 terraform-format:
 	$(info *********** cleaning terraform files syntax and generating terraform documentation ***********)
@@ -176,7 +178,7 @@ endif
 # END OF TERRAFORM SECTION
 ###################################
 # PACKER SECTION
-ifeq (, $(shell which packer))
+ifneq (yes, $(shell  ./tools/detect_packer.sh ))
 ## PACKER IS NOT PRESENT
 warn-packer-missing:
 	$(warning WARNING: packer not installed, visit https://learn.hashicorp.com/tutorials/packer/get-started-install-cli)
@@ -193,8 +195,9 @@ else
 ## PACKER IS PRESENT
 warn-packer-missing:
 
-PK_VERSION_CHECK=$(shell expr `packer version | cut -f 2- -d ' ' | cut -c 2- | head -n1` \>= $(MIN_PACKER_VERSION))
-ifneq ("$(PK_VERSION_CHECK)", "1")
+PK_VERSION=$(shell packer version | cut -f 2- -d ' ' | head -n1)
+PK_VERSION_CHECK=$(shell ./tools/version_check.sh $(PK_VERSION) $(MIN_PACKER_VERSION))
+ifneq ("$(PK_VERSION_CHECK)", "yes")
 ### WRONG PACKER VERSION, MAY ALSO MEAN THE USER HAS SOME OTHER PACKER TOOL
 warn-packer-version:
 	$(warning WARNING: packer version must be greater than $(MIN_PACKER_VERSION), update at https://learn.hashicorp.com/tutorials/packer/get-started-install-cli)

README.md

@@ -33,7 +33,7 @@ If a self directed path is preferred, you can use the following commands to
 build the `ghpc` binary:
 
 ```shell
-git clone git@github.com:GoogleCloudPlatform/hpc-toolkit.git
+git clone https://github.com/GoogleCloudPlatform/hpc-toolkit
 cd hpc-toolkit
 make
 ./ghpc --version
@@ -247,22 +247,19 @@ the same as the source module, for example the
 A hidden directory containing meta information and backups is also created and
 named `.ghpc`.
 
-From the [hpc-cluster-small.yaml example](./examples/hpc-cluster-small.yaml), we
+From the [hpc-slurm.yaml example](./examples/hpc-slurm.yaml), we
 get the following deployment directory:
 
 ```text
-hpc-small/
+hpc-slurm/
   primary/
     main.tf
-    variables.tf
-    terraform.tfvars
     modules/
-      filestore/
-      SchedMD-slurm-on-gcp-controller/
-      SchedMD-slurm-on-gcp-login-node/
-      SchedMD-slurm-on-gcp-partition/
-      vpc/
-    .ghpc/
+    providers.tf
+    terraform.tfvars
+    variables.tf
+    versions.tf
+  .ghpc/
 ```
 
 ## Dependencies

cmd/create.go

@@ -24,16 +24,18 @@ import (
 	"hpc-toolkit/pkg/modulewriter"
 	"log"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/zclconf/go-cty/cty"
+	"gopkg.in/yaml.v3"
 )
 
 const msgCLIVars = "Comma-separated list of name=value variables to override YAML configuration. Can be used multiple times."
 const msgCLIBackendConfig = "Comma-separated list of name=value variables to set Terraform backend configuration. Can be used multiple times."
 
 func init() {
-	createCmd.Flags().StringVarP(&bpFilename, "config", "c", "",
-		"HPC Environment Blueprint file to be used to create an HPC deployment dir.")
+	createCmd.Flags().StringVarP(&bpFilenameDeprecated, "config", "c", "", "")
 	cobra.CheckErr(createCmd.Flags().MarkDeprecated("config",
 		"please see the command usage for more details."))
 
@@ -52,9 +54,9 @@ func init() {
 }
 
 var (
-	bpFilename   string
-	outputDir    string
-	cliVariables []string
+	bpFilenameDeprecated string
+	outputDir            string
+	cliVariables         []string
 
 	cliBEConfigVars     []string
 	overwriteDeployment bool
@@ -64,52 +66,114 @@ var (
 	skipValidatorsDesc  = "Validators to skip"
 
 	createCmd = &cobra.Command{
-		Use:   "create BLUEPRINT_NAME",
-		Short: "Create a new deployment.",
-		Long:  "Create a new deployment based on a provided blueprint.",
-		Run:   runCreateCmd,
-		Args:  cobra.ExactArgs(1),
+		Use:               "create BLUEPRINT_NAME",
+		Short:             "Create a new deployment.",
+		Long:              "Create a new deployment based on a provided blueprint.",
+		Run:               runCreateCmd,
+		Args:              cobra.ExactArgs(1),
+		ValidArgsFunction: filterYaml,
 	}
 )
 
 func runCreateCmd(cmd *cobra.Command, args []string) {
-	if bpFilename == "" {
-		if len(args) == 0 {
-			fmt.Println(cmd.UsageString())
-			return
+	dc := expandOrDie(args[0])
+	if err := modulewriter.WriteDeployment(dc, outputDir, overwriteDeployment); err != nil {
+		var target *modulewriter.OverwriteDeniedError
+		if errors.As(err, &target) {
+			fmt.Printf("\n%s\n", err.Error())
+			os.Exit(1)
+		} else {
+			log.Fatal(err)
 		}
-
-		bpFilename = args[0]
 	}
+}
 
-	deploymentConfig, err := config.NewDeploymentConfig(bpFilename)
+func expandOrDie(path string) config.DeploymentConfig {
+	dc, err := config.NewDeploymentConfig(path)
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err := deploymentConfig.SetCLIVariables(cliVariables); err != nil {
+	// Set properties from CLI
+	if err := setCLIVariables(&dc.Config, cliVariables); err != nil {
 		log.Fatalf("Failed to set the variables at CLI: %v", err)
 	}
-	if err := deploymentConfig.SetBackendConfig(cliBEConfigVars); err != nil {
+	if err := setBackendConfig(&dc.Config, cliBEConfigVars); err != nil {
 		log.Fatalf("Failed to set the backend config at CLI: %v", err)
 	}
-	if err := deploymentConfig.SetValidationLevel(validationLevel); err != nil {
+	if err := setValidationLevel(&dc.Config, validationLevel); err != nil {
 		log.Fatal(err)
 	}
-	if err := skipValidators(&deploymentConfig); err != nil {
+	if err := skipValidators(&dc); err != nil {
 		log.Fatal(err)
 	}
-	if err := deploymentConfig.ExpandConfig(); err != nil {
+	if dc.Config.GhpcVersion != "" {
+		fmt.Printf("ghpc_version setting is ignored.")
+	}
+	dc.Config.GhpcVersion = GitCommitInfo
+
+	// Expand the blueprint
+	if err := dc.ExpandConfig(); err != nil {
 		log.Fatal(err)
 	}
-	if err := modulewriter.WriteDeployment(deploymentConfig, outputDir, overwriteDeployment); err != nil {
-		var target *modulewriter.OverwriteDeniedError
-		if errors.As(err, &target) {
-			fmt.Printf("\n%s\n", err.Error())
-			os.Exit(1)
-		} else {
-			log.Fatal(err)
+
+	return dc
+}
+
+func setCLIVariables(bp *config.Blueprint, s []string) error {
+	for _, cliVar := range s {
+		arr := strings.SplitN(cliVar, "=", 2)
+
+		if len(arr) != 2 {
+			return fmt.Errorf("invalid format: '%s' should follow the 'name=value' format", cliVar)
+		}
+		// Convert the variable's string literal to its equivalent default type.
+		key := arr[0]
+		var v config.YamlValue
+		if err := yaml.Unmarshal([]byte(arr[1]), &v); err != nil {
+			return fmt.Errorf("invalid input: unable to convert '%s' value '%s' to known type", key, arr[1])
 		}
+		bp.Vars.Set(key, v.Unwrap())
 	}
+	return nil
+}
+
+func setBackendConfig(bp *config.Blueprint, s []string) error {
+	if len(s) == 0 {
+		return nil // no op
+	}
+	be := config.TerraformBackend{Type: "gcs"}
+	for _, config := range s {
+		arr := strings.SplitN(config, "=", 2)
+
+		if len(arr) != 2 {
+			return fmt.Errorf("invalid format: '%s' should follow the 'name=value' format", config)
+		}
+
+		key, value := arr[0], arr[1]
+		switch key {
+		case "type":
+			be.Type = value
+		default:
+			be.Configuration.Set(key, cty.StringVal(value))
+		}
+	}
+	bp.TerraformBackendDefaults = be
+	return nil
+}
+
+// SetValidationLevel allows command-line tools to set the validation level
+func setValidationLevel(bp *config.Blueprint, s string) error {
+	switch s {
+	case "ERROR":
+		bp.ValidationLevel = config.ValidationError
+	case "WARNING":
+		bp.ValidationLevel = config.ValidationWarning
+	case "IGNORE":
+		bp.ValidationLevel = config.ValidationIgnore
+	default:
+		return fmt.Errorf("invalid validation level (\"ERROR\", \"WARNING\", \"IGNORE\")")
+	}
+	return nil
 }
 
 func skipValidators(dc *config.DeploymentConfig) error {
@@ -123,3 +187,10 @@ func skipValidators(dc *config.DeploymentConfig) error {
 	}
 	return nil
 }
+
+func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if len(args) != 0 {
+		return nil, cobra.ShellCompDirectiveNoFileComp
+	}
+	return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt
+}