Merge pull request #1551 from GoogleCloudPlatform/release-candidate

Release v1.20.0

Makefile

@@ -10,6 +10,7 @@ MIN_GOLANG_VERSION=1.18 # for building ghpc
         terraform-format packer-format \
         check-tflint check-pre-commit
 
+SHELL=/bin/bash -o pipefail
 ENG = ./cmd/... ./pkg/...
 TERRAFORM_FOLDERS=$(shell find ./modules ./community/modules ./tools -type f -name "*.tf" -not -path '*/\.*' -exec dirname "{}" \; | sort -u)
 PACKER_FOLDERS=$(shell find ./modules ./community/modules ./tools -type f -name "*.pkr.hcl" -not -path '*/\.*' -exec dirname "{}" \; | sort -u)

cmd/create.go

@@ -23,7 +23,7 @@ import (
 	"hpc-toolkit/pkg/config"
 	"hpc-toolkit/pkg/modulewriter"
 	"log"
-	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -77,19 +77,27 @@ var (
 
 func runCreateCmd(cmd *cobra.Command, args []string) {
 	dc := expandOrDie(args[0])
-	if err := modulewriter.WriteDeployment(dc, outputDir, overwriteDeployment); err != nil {
-		var target *modulewriter.OverwriteDeniedError
-		if errors.As(err, &target) {
-			fmt.Printf("\n%s\n", err.Error())
-			os.Exit(1)
-		} else {
-			log.Fatal(err)
-		}
-	}
+	deplName, err := dc.Config.DeploymentName()
+	cobra.CheckErr(err)
+	deplDir := filepath.Join(outputDir, deplName)
+	cobra.CheckErr(modulewriter.WriteDeployment(dc, deplDir, overwriteDeployment))
+
+	fmt.Println("To deploy your infrastructure please run:")
+	fmt.Println()
+	fmt.Printf("./ghpc deploy %s\n", deplDir)
+	fmt.Println()
+	printAdvancedInstructionsMessage(deplDir)
+}
+
+func printAdvancedInstructionsMessage(deplDir string) {
+	fmt.Println("Find instructions for cleanly destroying infrastructure and advanced manual")
+	fmt.Println("deployment instructions at:")
+	fmt.Println()
+	fmt.Printf("%s\n", modulewriter.InstructionsPath(deplDir))
 }
 
 func expandOrDie(path string) config.DeploymentConfig {
-	dc, err := config.NewDeploymentConfig(path)
+	dc, ctx, err := config.NewDeploymentConfig(path)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -113,12 +121,30 @@ func expandOrDie(path string) config.DeploymentConfig {
 
 	// Expand the blueprint
 	if err := dc.ExpandConfig(); err != nil {
-		log.Fatal(err)
+		log.Fatal(renderError(err, ctx))
 	}
 
 	return dc
 }
 
+func renderError(err error, ctx config.YamlCtx) string {
+	var be config.BpError
+	if errors.As(err, &be) {
+		if pos, ok := ctx.Pos(be.Path); ok {
+			return renderRichError(be.Err, pos, ctx)
+		}
+	}
+	return err.Error()
+}
+
+func renderRichError(err error, pos config.Pos, ctx config.YamlCtx) string {
+	return fmt.Sprintf(`
+Error: %s
+on line %d, column %d:
+%d: %s
+`, err, pos.Line, pos.Column, pos.Line, ctx.Lines[pos.Line-1])
+}
+
 func setCLIVariables(bp *config.Blueprint, s []string) error {
 	for _, cliVar := range s {
 		arr := strings.SplitN(cliVar, "=", 2)

cmd/create_test.go

@@ -15,6 +15,7 @@
 package cmd
 
 import (
+	"errors"
 	"hpc-toolkit/pkg/config"
 
 	"github.com/zclconf/go-cty/cty"
@@ -128,3 +129,31 @@ func (s *MySuite) TestValidationLevels(c *C) {
 
 	c.Check(setValidationLevel(&bp, "INVALID"), NotNil)
 }
+
+func (s *MySuite) TestRenderError(c *C) {
+	{ // simple
+		err := errors.New("arbuz")
+		got := renderError(err, config.YamlCtx{})
+		c.Check(got, Equals, "arbuz")
+	}
+	{ // has pos, but context is missing
+		ctx := config.NewYamlCtx([]byte(``))
+		pth := config.Root.Vars.Dot("kale")
+		err := config.BpError{Path: pth, Err: errors.New("arbuz")}
+		got := renderError(err, ctx)
+		c.Check(got, Equals, "vars.kale: arbuz")
+	}
+	{ // has pos, has context
+		ctx := config.NewYamlCtx([]byte(`
+vars:
+  kale: dos`))
+		pth := config.Root.Vars.Dot("kale")
+		err := config.BpError{Path: pth, Err: errors.New("arbuz")}
+		got := renderError(err, ctx)
+		c.Check(got, Equals, `
+Error: arbuz
+on line 3, column 9:
+3:   kale: dos
+`)
+	}
+}

cmd/deploy.go

@@ -18,6 +18,7 @@ package cmd
 import (
 	"fmt"
 	"hpc-toolkit/pkg/config"
+	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/shell"
 	"log"
 	"path/filepath"
@@ -48,7 +49,7 @@ var (
 		Args:              cobra.MatchAll(cobra.ExactArgs(1), checkDir),
 		ValidArgsFunction: matchDirs,
 		PreRunE:           parseDeployArgs,
-		RunE:              runDeployCmd,
+		Run:               runDeployCmd,
 		SilenceUsage:      true,
 	}
 )
@@ -72,40 +73,33 @@ func getApplyBehavior(autoApprove bool) shell.ApplyBehavior {
 	return shell.PromptBeforeApply
 }
 
-func runDeployCmd(cmd *cobra.Command, args []string) error {
+func runDeployCmd(cmd *cobra.Command, args []string) {
 	expandedBlueprintFile := filepath.Join(artifactsDir, expandedBlueprintFilename)
-	dc, err := config.NewDeploymentConfig(expandedBlueprintFile)
-	if err != nil {
-		return err
-	}
-
-	if err := shell.ValidateDeploymentDirectory(dc.Config.DeploymentGroups, deploymentRoot); err != nil {
-		return err
-	}
+	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	cobra.CheckErr(err)
+	cobra.CheckErr(shell.ValidateDeploymentDirectory(dc.Config.DeploymentGroups, deploymentRoot))
 
 	for _, group := range dc.Config.DeploymentGroups {
 		groupDir := filepath.Join(deploymentRoot, string(group.Name))
-		if err = shell.ImportInputs(groupDir, artifactsDir, expandedBlueprintFile); err != nil {
-			return err
-		}
+		cobra.CheckErr(shell.ImportInputs(groupDir, artifactsDir, expandedBlueprintFile))
 
 		var err error
 		switch group.Kind {
 		case config.PackerKind:
 			// Packer groups are enforced to have length 1
-			moduleDir := filepath.Join(groupDir, string(group.Modules[0].ID))
+			subPath, e := modulewriter.DeploymentSource(group.Modules[0])
+			cobra.CheckErr(e)
+			moduleDir := filepath.Join(groupDir, subPath)
 			err = deployPackerGroup(moduleDir)
 		case config.TerraformKind:
 			err = deployTerraformGroup(groupDir)
 		default:
 			err = fmt.Errorf("group %s is an unsupported kind %s", groupDir, group.Kind.String())
 		}
-		if err != nil {
-			return err
-		}
-
+		cobra.CheckErr(err)
 	}
-	return nil
+	fmt.Println("\n###############################")
+	printAdvancedInstructionsMessage(deploymentRoot)
 }
 
 func deployPackerGroup(moduleDir string) error {

cmd/destroy.go

@@ -64,7 +64,7 @@ func parseDestroyArgs(cmd *cobra.Command, args []string) error {
 
 func runDestroyCmd(cmd *cobra.Command, args []string) error {
 	expandedBlueprintFile := filepath.Join(artifactsDir, expandedBlueprintFilename)
-	dc, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
 	if err != nil {
 		return err
 	}

cmd/export.go

@@ -87,7 +87,7 @@ func runExportCmd(cmd *cobra.Command, args []string) error {
 	}
 
 	expandedBlueprintFile := filepath.Join(artifactsDir, expandedBlueprintFilename)
-	dc, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
 	if err != nil {
 		return err
 	}

cmd/import.go

@@ -51,7 +51,7 @@ func runImportCmd(cmd *cobra.Command, args []string) error {
 	}
 
 	expandedBlueprintFile := filepath.Join(artifactsDir, expandedBlueprintFilename)
-	dc, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -50,13 +50,16 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.19.1",
+		Version:     "v1.20.0",
 		Annotations: annotation,
 	}
 )
 
 // Execute the root command
 func Execute() error {
+	// Don't prefix messages with data & time to improve readability.
+	// See https://pkg.go.dev/log#pkg-constants
+	log.SetFlags(0)
 
 	mismatch, branch, hash, dir := checkGitHashMismatch()
 	if mismatch {
@@ -125,6 +128,9 @@ func hpcToolkitRepo() (repo *git.Repository, dir string, err error) {
 	// found. If it's the hpc-toolkit repo, return it.
 	// repo := new(git.Repository)
 	dir, err = os.Getwd()
+	if err != nil {
+		return nil, "", err
+	}
 	subdir := filepath.Dir(dir)
 	o := git.PlainOpenOptions{DetectDotGit: true}
 	repo, err = git.PlainOpenWithOptions(dir, &o)
@@ -168,8 +174,5 @@ func hpcToolkitRepo() (repo *git.Repository, dir string, err error) {
 func isHpcToolkitRepo(r git.Repository) bool {
 	h := plumbing.NewHash(GitInitialHash)
 	_, err := r.CommitObject(h)
-	if err == nil {
-		return true
-	}
-	return false
+	return err == nil
 }

cmd/root_test.go

@@ -199,7 +199,7 @@ func checkPathsEqual(c *C, a, b string) {
 	if err != nil {
 		c.Fatal(err)
 	}
-	b, err = filepath.EvalSymlinks(a)
+	b, err = filepath.EvalSymlinks(b)
 	if err != nil {
 		c.Fatal(err)
 	}
@@ -241,6 +241,9 @@ func initTestRepo(path string) (repo *git.Repository, initHash plumbing.Hash, er
 	}
 
 	initHash, err = commit("Init")
+	if err != nil {
+		return
+	}
 	_, err = commit("Last")
 	return
 }

community/examples/hpc-slurm-chromedesktop.yaml

@@ -54,7 +54,7 @@ deployment_groups:
       disable_public_ips: false
       instance_image:
         family: slurm-gcp-5-7-debian-11
-        project: projects/schedmd-slurm-public/global/images/family
+        project: schedmd-slurm-public
       guest_accelerator:
       - type: nvidia-tesla-t4-vws
         count: 1

community/examples/hpc-slurm-ubuntu2004.yaml

@@ -25,7 +25,7 @@ vars:
     # Please refer to the following link for the latest images:
     # https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems
     family: slurm-gcp-5-7-ubuntu-2004-lts
-    project: projects/schedmd-slurm-public/global/images/family
+    project: schedmd-slurm-public
 
 
 deployment_groups: