Skip to content

Commit

Permalink
enhance NodeExporter metrics
Browse files Browse the repository at this point in the history 
This adds a "hostname" label to NodeExporter metrics
to make the identification of the node of origin a little easier
than just having the IP address.

As a side effect of needing a little different parsing for
NodeExporter and Kepler ScrapeConfigs, the code to get compute
node IPs from ansible facts and from IPsets is now executed
only once (in comparison to executing it twice), which greatly
reduces k8s api calls.
  • Loading branch information
@vyzigold
vyzigold authored and openshift-cherrypick-robot committed Nov 19, 2024
1 parent 7e172ac commit ee7a0fc
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 30 deletions.
85 changes: 65 additions & 20 deletions controllers/metricstorage_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@ type MetricStorageReconciler struct {
Cache cache.Cache
}

// ConnectionInfo holds information about connection to a compute node
type ConnectionInfo struct {
IP string
Hostname string
TLS bool
}

// GetLogger returns a logger object with a prefix of "conroller.name" and aditional controller context fields
func (r *MetricStorageReconciler) GetLogger(ctx context.Context) logr.Logger {
return log.FromContext(ctx).WithName("Controllers").WithName("MetricStorage")
Expand Down Expand Up @@ -506,7 +513,7 @@ func (r *MetricStorageReconciler) createServiceScrapeConfig(
log logr.Logger,
description string,
serviceName string,
targets []string,
targets interface{},
tlsEnabled bool,
) error {
scrapeConfig := &monv1alpha1.ScrapeConfig{
Expand Down Expand Up @@ -593,30 +600,31 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
return ctrl.Result{}, err
}

// ScrapeConfigs for NodeExporters
endpointsNonTLS, endpointsTLS, err := getMetricExporterTargets(instance, helper, telemetryv1.DefaultNodeExporterPort)
connectionInfo, err := getComputeNodesConnectionInfo(instance, helper)
if err != nil {
Log.Info(fmt.Sprintf("Cannot get node exporter targets. Scrape configs not created. Error: %s", err))
Log.Info(fmt.Sprintf("Cannot get compute node connection info. Scrape configs not created. Error: %s", err))
}

// ScrapeConfigs for NodeExporters
neTargetsTLS, neTargetsNonTLS := getNodeExporterTargets(connectionInfo)
// ScrapeConfig for non-tls nodes
neServiceName := fmt.Sprintf("%s-node-exporter", telemetry.ServiceName)
err = r.createServiceScrapeConfig(ctx, instance, Log, "Node Exporter",
neServiceName, endpointsNonTLS, false)
neServiceName, neTargetsNonTLS, false)
if err != nil {
return ctrl.Result{}, err
}

// ScrapeConfig for tls nodes
neServiceName = fmt.Sprintf("%s-node-exporter-tls", telemetry.ServiceName)
err = r.createServiceScrapeConfig(ctx, instance, Log, "Node Exporter",
neServiceName, endpointsTLS, true)
neServiceName, neTargetsTLS, true)
if err != nil {
return ctrl.Result{}, err
}

// kepler scrape endpoints
_, keplerEndpoints, err := getMetricExporterTargets(instance, helper, telemetryv1.DefaultKeplerPort)
keplerEndpoints, _ := getKeplerTargets(connectionInfo)
if err != nil {
Log.Info(fmt.Sprintf("Cannot get Kepler targets. Scrape configs not created. Error: %s", err))
}
Expand All @@ -633,6 +641,37 @@ func (r *MetricStorageReconciler) createScrapeConfigs(
return ctrl.Result{}, err
}

func getNodeExporterTargets(nodes []ConnectionInfo) ([]metricstorage.LabeledTarget, []metricstorage.LabeledTarget) {
tls := []metricstorage.LabeledTarget{}
nonTLS := []metricstorage.LabeledTarget{}
for _, node := range nodes {
target := metricstorage.LabeledTarget{
IP: fmt.Sprintf("%s:%d", node.IP, telemetryv1.DefaultNodeExporterPort),
Hostname: node.Hostname,
}
if node.TLS {
tls = append(tls, target)
} else {
nonTLS = append(nonTLS, target)
}
}
return tls, nonTLS
}

func getKeplerTargets(nodes []ConnectionInfo) ([]string, []string) {
tls := []string{}
nonTLS := []string{}
for _, node := range nodes {
target := fmt.Sprintf("%s:%d", node.IP, telemetryv1.DefaultKeplerPort)
if node.TLS {
tls = append(tls, target)
} else {
nonTLS = append(nonTLS, target)
}
}
return tls, nonTLS
}

func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, instance *telemetryv1.MetricStorage, eventHandler handler.EventHandler) (ctrl.Result, error) {
Log := r.GetLogger(ctx)
uiPluginObj := &obsui.UIPlugin{
Expand Down Expand Up @@ -795,26 +834,24 @@ func (r *MetricStorageReconciler) ensureWatches(
return err
}

func getMetricExporterTargets(
func getComputeNodesConnectionInfo(
instance *telemetryv1.MetricStorage,
helper *helper.Helper,
defaultMetricExporterPort int32,
) ([]string, []string, error) {
) ([]ConnectionInfo, error) {
ipSetList, err := getIPSetList(instance, helper)
if err != nil {
return []string{}, []string{}, err
return []ConnectionInfo{}, err
}
inventorySecretList, err := getInventorySecretList(instance, helper)
if err != nil {
return []string{}, []string{}, err
return []ConnectionInfo{}, err
}
var address string
addressesNonTLS := []string{}
addressesTLS := []string{}
connectionInfo := []ConnectionInfo{}
for _, secret := range inventorySecretList.Items {
inventory, err := ansible.UnmarshalYAML(secret.Data["inventory"])
if err != nil {
return []string{}, []string{}, err
return []ConnectionInfo{}, err
}
nodeSetGroup := inventory.Groups[secret.Labels["openstackdataplanenodeset"]]
containsTelemetry := false
Expand Down Expand Up @@ -842,20 +879,28 @@ func getMetricExporterTargets(
address, _ = getAddressFromAnsibleHost(&item)
} else {
// we were unable to find an IP or HostName for a node, so we do not go further
return addressesNonTLS, addressesTLS, fmt.Errorf("failed to find an IP or HostName for node %s", name)
return connectionInfo, fmt.Errorf("failed to find an IP or HostName for node %s", name)
}
if address == "" {
// we were unable to find an IP or HostName for a node, so we do not go further
return addressesNonTLS, addressesTLS, fmt.Errorf("failed to find an IP or HostName for node %s", name)
return connectionInfo, fmt.Errorf("failed to find an IP or HostName for node %s", name)
}
if TLSEnabled, ok := nodeSetGroup.Vars["edpm_tls_certs_enabled"].(bool); ok && TLSEnabled {
addressesTLS = append(addressesTLS, fmt.Sprintf("%s:%d", address, defaultMetricExporterPort))
connectionInfo = append(connectionInfo, ConnectionInfo{
IP: address,
Hostname: name,
TLS: true,
})
} else {
addressesNonTLS = append(addressesNonTLS, fmt.Sprintf("%s:%d", address, defaultMetricExporterPort))
connectionInfo = append(connectionInfo, ConnectionInfo{
IP: address,
Hostname: name,
TLS: false,
})
}
}
}
return addressesNonTLS, addressesTLS, nil
return connectionInfo, nil
}

func getIPSetList(instance *telemetryv1.MetricStorage, helper *helper.Helper) (*infranetworkv1.IPSetList, error) {
Expand Down
39 changes: 29 additions & 10 deletions pkg/metricstorage/scrape_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,16 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

type LabeledTarget struct {
IP string
Hostname string
}

// ScrapeConfig creates a ScrapeConfig CR
func ScrapeConfig(
instance *telemetryv1.MetricStorage,
labels map[string]string,
targets []string,
targets interface{},
tlsEnabled bool,
) *monv1alpha1.ScrapeConfig {
var scrapeInterval monv1.Duration
Expand All @@ -46,10 +51,28 @@ func ScrapeConfig(
scrapeInterval = telemetryv1.DefaultScrapeInterval
}

sort.Strings(targets)
var convertedTargets []monv1alpha1.Target
for _, t := range targets {
convertedTargets = append(convertedTargets, monv1alpha1.Target(t))
var staticConfigs []monv1alpha1.StaticConfig
if ips, ok := targets.([]string); ok {
sort.Strings(ips)
var convertedTargets []monv1alpha1.Target
for _, t := range ips {
convertedTargets = append(convertedTargets, monv1alpha1.Target(t))
}
staticConfigs = append(staticConfigs, monv1alpha1.StaticConfig{
Targets: convertedTargets,
})
} else if labeledTargets, ok := targets.([]LabeledTarget); ok {
sort.Slice(labeledTargets, func(i, j int) bool {
return labeledTargets[i].IP < labeledTargets[j].IP
})
for _, t := range labeledTargets {
staticConfigs = append(staticConfigs, monv1alpha1.StaticConfig{
Targets: []monv1alpha1.Target{monv1alpha1.Target(t.IP)},
Labels: map[monv1.LabelName]string{
"hostname": t.Hostname,
},
})
}
}

scrapeConfig := &monv1alpha1.ScrapeConfig{
Expand Down Expand Up @@ -82,11 +105,7 @@ func ScrapeConfig(
},
},
ScrapeInterval: &scrapeInterval,
StaticConfigs: []monv1alpha1.StaticConfig{
{
Targets: convertedTargets,
},
},
StaticConfigs: staticConfigs,
},
}

Expand Down

0 comments on commit ee7a0fc

Please sign in to comment.