Merge pull request #140 from leanix/master

HEL1270 Connector Logs in iHub

CHANGELOG.md

@@ -1,5 +1,8 @@
 # LeanIX Kubernetes Connector Changelog
 
+## Release 2022-01-03 - 6.3.0
+* Connector Logs can be accessed in iHub (w.r.t specific dataSource name) `Show Log` button for troubleshooting and monitoring purpose.
+
 ## Release 2022-01-03 - 6.2.0
 * To ensure support for multiple helm release on same cluster, all the existing K8s workloads are suffixed with the Helm release name.
 * Support multiple workspace ID for same cluster via different helm releases

Makefile

@@ -1,7 +1,7 @@
 PROJECT ?= leanix-k8s-connector
 DOCKER_NAMESPACE ?= leanixacrpublic.azurecr.io
 
-VERSION := 6.2.0
+VERSION := 6.3.0
 FULL_VERSION := $(VERSION)-$(shell git describe --tags --always)
 
 IMAGE := $(DOCKER_NAMESPACE)/$(PROJECT):$(VERSION)

README.md

@@ -553,7 +553,7 @@ Issue `kubectl delete jobs.batch leanix-k8s-connector-1563961200` and you should
 - This migration involves changes to make the helm configurations much simpler. The operator does not have to worry about such parameters while scanning the data source.
   - The following helm values can not be set as parameters anymore. Therefore, no need to use them either in helm commands or values.yaml.
     - `args.clustername` : The iHUB connector configuration for the k8s integration is extended to capture the 'clusterName'. 
-    - `args.connectorID`, `args.connectorVersion`, `args.processingMode` : Here these values are set by default and no need to set with helm commands or values.yaml 
+    - `args.connectorID`, `args.connectorVersion`, `args.processingMode` : Here these values are set by default and no need to set with helm commands or values.yaml 
 
 ### 4.0.0 to 5.0.0
 - New flag `enableCustomStorage` is introduced. This flags allows to disable the option to upload LDIF to custom storage backend. Disabling the flag will not affect the functionality of the connector.
@@ -578,6 +578,8 @@ e.g `--set args.storageBackend=file`
 
 | Release date | Connector version | Integration version | Helm chart version | Container image tag |
 | :----------: | :---------------: | :-----------------: | :----------------: | :-----------------: |
+|  2022-01-03  |       6.3.0       |        1.0.0        |       6.3.0        |        6.3.0        |
+|  2022-01-03  |       6.2.0       |        1.0.0        |       6.2.0        |        6.2.0        |
 |  2021-12-08  |       6.1.0       |        1.0.0        |       6.1.0        |        6.1.0        |
 |  2021-12-06  |       6.0.3       |        1.0.0        |       6.0.3        |        6.0.3        |
 |  2021-12-19  |       6.0.2       |        1.0.0        |       6.0.2        |        6.0.2        |

cmd/leanix-k8s-connector/main.go

@@ -56,21 +56,34 @@ func main() {
 	stdoutLogger, debugLogBuffer := initLogger()
 	err := parseFlags()
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
 	}
 	enableVerbose(stdoutLogger, viper.GetBool(verboseFlag))
 
+	// use the current context in kubeconfig
+	startResponse, err := KubernetesScan(debugLogBuffer)
+	if err != nil {
+		log.Error(err)
+	}
+	log.Info("Uploading connector logs to iHub")
+	err = storage.UploadConnectorLog(startResponse.ConnectorLoggingUrl, debugLogBuffer.Bytes())
+	if err != nil {
+		log.Error(err)
+	}
+}
+
+func KubernetesScan(debugLogBuffer *bytes.Buffer) (response *leanix.SelfStartResponse, err error) {
 	log.Info("----------Attempting to Self Start via Integration Hub----------")
 
 	accessToken, err := leanix.Authenticate(viper.GetString(integrationAPIFqdnFlag), viper.GetString(integrationAPITokenFlag))
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
 	}
 	log.Info("Integration Hub authentication successful.")
 	startResponse, err := leanix.SelfStartRun(viper.GetString(integrationAPIFqdnFlag), accessToken, viper.GetString(integrationAPIDatasourceNameFlag))
 	if err != nil {
 		log.Info("Failed to start Integration Hub. Terminating..")
-		log.Fatal(err)
+		log.Error(err)
 		return
 	}
 	log.Info("Getting connector config...")
@@ -95,43 +108,44 @@ func main() {
 	if viper.GetBool(localFlag) {
 		config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(homedir.HomeDir(), ".kube", "config"))
 		if err != nil {
-			log.Fatal(err)
+			return nil, err
 		}
 	} else {
-		// use the current context in kubeconfig
+
 		config, err = restclient.InClusterConfig()
 		if err != nil {
-			log.Fatalf("Failed to load kube config. Running in Kubernetes?\n%s", err)
+			log.Errorf("Failed to load kube config. Running in Kubernetes?\n%s", err)
+			return nil, err
 		}
 	}
 
 	log.Debugf("Kubernetes master from config: %s", config.Host)
 
 	kubernetesAPI, err := kubernetes.NewAPI(config)
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	dynClient, err := dynamic.NewForConfig(config)
 	if err != nil {
-		log.Fatal(err)
+		log.Error(err)
 	}
 
 	log.Debug("Get blacklist namespaces list...")
 	blacklistedNamespacesList := viper.GetStringSlice(blacklistNamespacesFlag)
 	blacklistedNamespaces, err := kubernetesAPI.Namespaces(blacklistedNamespacesList)
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	log.Debug("Getting blacklist namespaces list done.")
 	log.Infof("Namespace blacklist: %v", reflect.ValueOf(blacklistedNamespaces).MapKeys())
 
 	resourcesList, err := ServerPreferredListableResources(kubernetesAPI.Client.Discovery())
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	groupVersionResources, err := discovery.GroupVersionResources(resourcesList)
 	if err != nil {
-		log.Panic(err)
+		return nil, err
 	}
 	_, err = leanix.UpdateInProgressStatus(startResponse.ProgressCallbackUrl, "Discovery of Version Resources is done. Moving on to mapping nodes")
 	if err != nil {
@@ -140,7 +154,7 @@ func main() {
 	log.Debug("Listing nodes...")
 	nodes, err := kubernetesAPI.Nodes()
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	log.Debug("Listing nodes done.")
 	log.Debug("Map nodes to Kubernetes object")
@@ -149,7 +163,7 @@ func main() {
 		nodes,
 	)
 	if err != nil {
-		log.Fatal(err)
+		return nil, err
 	}
 	kubernetesObjects := make([]mapper.KubernetesObject, 0)
 	kubernetesObjects = append(kubernetesObjects, *clusterKubernetesObject)
@@ -210,7 +224,7 @@ func main() {
 		}
 		instances, err := dynClient.Resource(gvr).List(context.Background(), metav1.ListOptions{})
 		if err != nil {
-			log.Panic(err)
+			return nil, err
 		}
 		for _, i := range instances.Items {
 			if _, ok := blacklistedNamespaces[i.GetNamespace()]; ok {
@@ -252,7 +266,8 @@ func main() {
 	ldifByte, err := storage.Marshal(ldif)
 	if err != nil {
 		_, err = leanix.UpdateFailedProgressStatus(startResponse.ProgressCallbackUrl, "Failed to marshal ldif")
-		log.Fatal(err)
+		log.Error(err)
+		return nil, err
 	}
 
 	if viper.GetBool(enableCustomStorageFlag) {
@@ -273,23 +288,17 @@ func main() {
 		uploader, err := storage.NewBackend(viper.GetString("storage-backend"), &azureOpts, &localFileOpts)
 		if err != nil {
 			_, err = leanix.UpdateFailedProgressStatus(startResponse.ProgressCallbackUrl, "Failed to create uploader for backend storage")
-			log.Fatal(err)
+			return nil, err
 		}
 		err = uploader.UploadLdif(ldifByte, storage.LdifFileName+viper.GetString(lxWorkspaceFlag)+storage.LdifFileExtension)
 		if err != nil {
-			log.Fatal(err)
+			log.Error(err)
 			_, err := leanix.UpdateFailedProgressStatus(startResponse.ProgressCallbackUrl, "Failed to upload ldif to backend storage configured storage backend - "+viper.GetString("storage-backend"))
 			if err != nil {
-				return
+				return nil, err
 			}
 		}
 
-		log.Info("Uploading log file to configured backend storage")
-		err = uploader.UploadLog(debugLogBuffer.Bytes(), storage.LogFileName+viper.GetString(lxWorkspaceFlag)+storage.LogFileExtension)
-		if err != nil {
-			log.Fatal(err)
-		}
-
 		_, err = leanix.UpdateInProgressStatus(startResponse.ProgressCallbackUrl, "Successfully uploaded ldif to configured storage backend - "+viper.GetString("storage-backend"))
 		if err != nil {
 			log.Infof("Failed to update progress[%s] to Integration Hub", leanix.IN_PROGRESS)
@@ -303,15 +312,16 @@ func main() {
 	if err != nil {
 		log.Debug("Failed to upload ldif to Integration Hub ldif SAS Url")
 		_, err := leanix.UpdateFailedProgressStatus(startResponse.ProgressCallbackUrl, "Failed to upload ldif to Integration Hub ldif SAS Url")
-		log.Fatal(err)
-		return
+		log.Error(err)
+		return nil, err
 	}
 	_, err = leanix.UpdateProgress(startResponse.ProgressCallbackUrl, leanix.FINISHED, "")
 	if err != nil {
 		log.Infof("Failed to update progress[%s] to Integration Hub", leanix.FINISHED)
 	}
 	log.Debug("-----------End-----------")
 	log.Info("-----------End-----------")
+	return startResponse, err
 }
 
 func ServerPreferredListableResources(d discovery.DiscoveryInterface) ([]*metav1.APIResourceList, error) {
@@ -339,6 +349,7 @@ func parseFlags() error {
 	// Let flags overwrite configs in viper
 	err := viper.BindPFlags(flag.CommandLine)
 	if err != nil {
+		log.Error(err)
 		return err
 	}
 	// Check for config values in env vars

helm/index.yaml

@@ -1,9 +1,19 @@
 apiVersion: v1
 entries:
   leanix-k8s-connector:
+  - apiVersion: v1
+    appVersion: 6.3.0
+    created: "2022-01-03T13:13:26.726066+01:00"
+    description: Retrieves information from Kubernetes cluster
+    digest: 44973672d6054cd0660a94e71b23bf9cf472f27bc532133981af6f6055e87424
+    kubeVersion: '>= 1.19.0-0'
+    name: leanix-k8s-connector
+    urls:
+    - leanix-k8s-connector-6.3.0.tgz
+    version: 6.3.0
   - apiVersion: v1
     appVersion: 6.2.0
-    created: "2021-12-23T14:40:59.713818+01:00"
+    created: "2022-01-03T13:13:26.723288+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 5774478e89bc749ba21ec7f2477bb348788f2bbf6e8b76956eee5c4739de9e46
     kubeVersion: '>= 1.19.0-0'
@@ -13,7 +23,7 @@ entries:
     version: 6.2.0
   - apiVersion: v1
     appVersion: 6.1.0
-    created: "2021-12-23T14:40:59.711566+01:00"
+    created: "2022-01-03T13:13:26.721701+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: eabf3e49a9d4a0722228cb24872e6dfcc7b9db6b764bc714a74c0941f0d3ea4f
     name: leanix-k8s-connector
@@ -22,7 +32,7 @@ entries:
     version: 6.1.0
   - apiVersion: v1
     appVersion: 6.0.3
-    created: "2021-12-23T14:40:59.710467+01:00"
+    created: "2022-01-03T13:13:26.720341+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 5f1be6a80fc173d45ca60f5da8937abc0fcb2cc14652d6149d9ff2e405568a71
     name: leanix-k8s-connector
@@ -31,7 +41,7 @@ entries:
     version: 6.0.3
   - apiVersion: v1
     appVersion: 6.0.2
-    created: "2021-12-23T14:40:59.709385+01:00"
+    created: "2022-01-03T13:13:26.718644+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: ccd112d1de601bfa796c4413ddc2b394f93ec4b66604338a736593168523b327
     name: leanix-k8s-connector
@@ -40,7 +50,7 @@ entries:
     version: 6.0.2
   - apiVersion: v1
     appVersion: 6.0.1
-    created: "2021-12-23T14:40:59.708288+01:00"
+    created: "2022-01-03T13:13:26.716584+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: f3b959e24bdbf55314fa6d0a05681ec41dd34b1493b026a3450bf147e62a4b1b
     name: leanix-k8s-connector
@@ -49,7 +59,7 @@ entries:
     version: 6.0.1
   - apiVersion: v1
     appVersion: 6.0.0
-    created: "2021-12-23T14:40:59.70717+01:00"
+    created: "2022-01-03T13:13:26.715001+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: c51f68b592c6f608a2f982e8c322280b8fc3aa4ac6ed0d823b2baf24b0877e6b
     name: leanix-k8s-connector
@@ -58,7 +68,7 @@ entries:
     version: 6.0.0
   - apiVersion: v1
     appVersion: 5.0.0
-    created: "2021-12-23T14:40:59.705136+01:00"
+    created: "2022-01-03T13:13:26.712156+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 549a753cf9875ca4eeaf5ecb5c64e76149b30bb8e82eee821f6c81cca27c37cf
     name: leanix-k8s-connector
@@ -67,7 +77,7 @@ entries:
     version: 5.0.0
   - apiVersion: v1
     appVersion: 4.0.0
-    created: "2021-12-23T14:40:59.704044+01:00"
+    created: "2022-01-03T13:13:26.710963+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: adf5c8ef024857c4a0845a5c659f27bd7f21cdf568fff3445a192425c44fc9a8
     name: leanix-k8s-connector
@@ -76,7 +86,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: 3.0.0
-    created: "2021-12-23T14:40:59.703176+01:00"
+    created: "2022-01-03T13:13:26.710195+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 2927aebc215a1c5175e986135e36a10101e90fa69e7ab22a183049d086f7eddb
     name: leanix-k8s-connector
@@ -85,7 +95,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: 2.0.0
-    created: "2021-12-23T14:40:59.702333+01:00"
+    created: "2022-01-03T13:13:26.709008+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 3481723d2374757038495117518c47f634054408ccbd55112a6260d5a15e18c9
     name: leanix-k8s-connector
@@ -94,7 +104,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: 2.0.0-beta7
-    created: "2021-12-23T14:40:59.701445+01:00"
+    created: "2022-01-03T13:13:26.707932+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: 0ade89d01236286f11ebf70290e2c4c82740fac11ec755be9dddd2375c74246f
     name: leanix-k8s-connector
@@ -103,11 +113,11 @@ entries:
     version: 2.0.0-beta7
   - apiVersion: v1
     appVersion: 1.1.0
-    created: "2021-12-23T14:40:59.700206+01:00"
+    created: "2022-01-03T13:13:26.706693+01:00"
     description: Retrieves information from Kubernetes cluster
     digest: aa3ee10eefe30e61e37058762b031bdee11cb9dd67afecdf6e831335d5c16e0f
     name: leanix-k8s-connector
     urls:
     - leanix-k8s-connector-1.0.0.tgz
     version: 1.0.0
-generated: "2021-12-23T14:40:59.698587+01:00"
+generated: "2022-01-03T13:13:26.705274+01:00"

helm/leanix-k8s-connector/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-appVersion: "6.2.0"
+appVersion: "6.3.0"
 description: Retrieves information from Kubernetes cluster
 name: leanix-k8s-connector
-version: 6.2.0
+version: 6.3.0
 kubeVersion: '>= 1.19.0-0'

pkg/leanix/integration_hub.go

@@ -17,6 +17,7 @@ type SelfStartResponse struct {
 	ConnectorConfiguration ConnectorConfiguration `json:"connectorConfiguration"`
 	LdifResultUrl          string                 `json:"ldifResultUrl"`
 	ProgressCallbackUrl    string                 `json:"progressCallbackUrl"`
+	ConnectorLoggingUrl    string                 `json:"connectorLoggingUrl"`
 }
 
 type ConnectorConfiguration struct {
@@ -80,7 +81,7 @@ func validateConnectorConfiguration(configuration ConnectorConfiguration) (bool,
 		return false, fmt.Errorf("INVALID CONNECTOR CONFIGURATION: RESOLVE LABEL CANNOT BE EMPTY IF THE RESOLVE STRATEGY IS 'LABEL'")
 	}
 
-	if configuration.ResolveStrategy == "label" && configuration.ClusterName == "" {
+	if configuration.ClusterName == "" {
 		return false, fmt.Errorf("INVALID CONNECTOR CONFIGURATION: CLUSTER NAME CANNOT BE EMPTY")
 	}
 

pkg/storage/azureblob.go

@@ -87,3 +87,15 @@ func UploadFileToSasUrl(blobSasUrl url.URL, content []byte) error {
 	}
 	return nil
 }
+
+func UploadConnectorLog(connectorLoggingUrl string, logs []byte) error {
+	parse, err := url.Parse(connectorLoggingUrl)
+	if err != nil {
+		return err
+	}
+	err = UploadFileToSasUrl(*parse, logs)
+	if err != nil {
+		return err
+	}
+	return nil
+}