Skip to content

Commit

Permalink
feat: Use PartialObjectMetadata for Configmaps and Secrets
Browse files Browse the repository at this point in the history
  • Loading branch information
@mrueg
mrueg committed Aug 7, 2024
1 parent f8aa7d9 commit cd9a7c2
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 17 deletions.
70 changes: 65 additions & 5 deletions internal/store/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ import (
policyv1 "k8s.io/api/policy/v1"
rbacv1 "k8s.io/api/rbac/v1"
storagev1 "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"

Expand All @@ -65,9 +67,10 @@ var _ ksmtypes.BuilderInterface = &Builder{}
// Builder helps to build store. It follows the builder pattern
// (https://en.wikipedia.org/wiki/Builder_pattern).
type Builder struct {
kubeClient clientset.Interface
customResourceClients map[string]interface{}
namespaces options.NamespaceList
kubeClient clientset.Interface
metadataOnlyKubeClient metadata.Interface
customResourceClients map[string]interface{}
namespaces options.NamespaceList
// namespaceFilter is inside fieldSelectorFilter
fieldSelectorFilter string
ctx context.Context
Expand All @@ -78,6 +81,7 @@ type Builder struct {
shard int32
totalShards int
buildStoresFunc ksmtypes.BuildStoresFunc
buildMetadataOnlyStoresFunc ksmtypes.BuildMetadataOnlyStoresFunc
buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
allowAnnotationsList map[string][]string
allowLabelsList map[string][]string
Expand Down Expand Up @@ -157,6 +161,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) {
b.kubeClient = c
}

// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder.
func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) {
b.metadataOnlyKubeClient = c
}

// WithCustomResourceClients sets the customResourceClients property of a Builder.
func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
b.customResourceClients = cs
Expand All @@ -178,6 +187,11 @@ func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
b.buildStoresFunc = f
}

// WithGenerateMetadataOnlyStoresFunc configures a custom generate custom resource store function
func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) {
b.buildMetadataOnlyStoresFunc = f
}

// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
b.buildCustomResourceStoresFunc = f
Expand All @@ -188,6 +202,11 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.buildStores
}

// DefaultGenerateMetadataOnlyStoresFunc returns default buildStores function
func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc {
return b.buildMetadataOnlyStores
}

// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function
func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
return b.buildCustomResourceStores
Expand Down Expand Up @@ -362,7 +381,7 @@ func availableResources() []string {
}

func (b *Builder) buildConfigMapStores() []cache.Store {
return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache)
return b.buildMetadataOnlyStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &metav1.PartialObjectMetadata{}, createConfigMapListWatch, b.useAPIServerCache)
}

func (b *Builder) buildCronJobStores() []cache.Store {
Expand Down Expand Up @@ -519,7 +538,8 @@ func (b *Builder) buildStores(
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
kubeClient := b.kubeClient
listWatcher := listWatchFunc(kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
return []cache.Store{store}
}
Expand All @@ -541,6 +561,46 @@ func (b *Builder) buildStores(
return stores
}

func (b *Builder) buildMetadataOnlyStores(
metricFamilies []generator.FamilyGenerator,
expectedType interface{},
listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher,
useAPIServerCache bool,
) []cache.Store {
metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)

if b.namespaces.IsAllNamespaces() {
store := metricsstore.NewMetricsStore(
familyHeaders,
composedMetricGenFuncs,
)
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
return []cache.Store{store}
}

stores := make([]cache.Store, 0, len(b.namespaces))
for _, ns := range b.namespaces {
store := metricsstore.NewMetricsStore(
familyHeaders,
composedMetricGenFuncs,
)
if b.fieldSelectorFilter != "" {
klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
}
listWatcher := listWatchFunc(b.metadataOnlyKubeClient, ns, b.fieldSelectorFilter)
b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
stores = append(stores, store)
}

return stores
}

// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores`
func (b *Builder) buildCustomResourceStores(resourceName string,
metricFamilies []generator.FamilyGenerator,
Expand Down
24 changes: 12 additions & 12 deletions internal/store/configmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ package store
import (
"context"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"
basemetrics "k8s.io/component-base/metrics"

Expand All @@ -43,7 +43,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.ALPHA,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
if len(allowAnnotationsList) == 0 {
return &metric.Family{}
}
Expand All @@ -65,7 +65,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
if len(allowLabelsList) == 0 {
return &metric.Family{}
}
Expand All @@ -87,7 +87,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(_ *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(_ *metav1.PartialObjectMetadata) *metric.Family {
return &metric.Family{
Metrics: []*metric.Metric{{
LabelKeys: []string{},
Expand All @@ -103,7 +103,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.STABLE,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
ms := []*metric.Metric{}

if !c.CreationTimestamp.IsZero() {
Expand All @@ -125,7 +125,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
metric.Gauge,
basemetrics.ALPHA,
"",
wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family {
return &metric.Family{
Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion),
}
Expand All @@ -134,22 +134,22 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
}
}

func createConfigMapListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher {
func createConfigMapListWatch(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher {
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
opts.FieldSelector = fieldSelector
return kubeClient.CoreV1().ConfigMaps(ns).List(context.TODO(), opts)
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).List(context.TODO(), opts)
},
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
opts.FieldSelector = fieldSelector
return kubeClient.CoreV1().ConfigMaps(ns).Watch(context.TODO(), opts)
return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).Watch(context.TODO(), opts)
},
}
}

func wrapConfigMapFunc(f func(*v1.ConfigMap) *metric.Family) func(interface{}) *metric.Family {
func wrapConfigMapFunc(f func(*metav1.PartialObjectMetadata) *metric.Family) func(interface{}) *metric.Family {
return func(obj interface{}) *metric.Family {
configMap := obj.(*v1.ConfigMap)
configMap := obj.(*metav1.PartialObjectMetadata)

metricFamily := f(configMap)

Expand Down
7 changes: 7 additions & 0 deletions pkg/app/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {

storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache)
storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc())
storeBuilder.WithGenerateMetadataOnlyStoresFunc(storeBuilder.DefaultGenerateMetadataOnlyStoresFunc())
proc.StartReaper()

storeBuilder.WithUtilOptions(opts)
Expand All @@ -275,6 +276,12 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
}
storeBuilder.WithKubeClient(kubeClient)

metadataOnlyKubeClient, err := util.CreateMetadataOnlyKubeClient(opts.Apiserver, opts.Kubeconfig)
if err != nil {
return fmt.Errorf("failed to create metadata-only client: %v", err)
}
storeBuilder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient)

storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil {
return fmt.Errorf("failed to set up annotations allowlist: %v", err)
Expand Down
16 changes: 16 additions & 0 deletions pkg/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/prometheus/client_golang/prometheus"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"

internalstore "k8s.io/kube-state-metrics/v2/internal/store"
Expand Down Expand Up @@ -84,6 +85,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) {
b.internal.WithKubeClient(c)
}

// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder.
func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) {
b.internal.WithMetadataOnlyKubeClient(c)
}

// WithCustomResourceClients sets the customResourceClients property of a Builder.
func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
b.internal.WithCustomResourceClients(cs)
Expand Down Expand Up @@ -120,6 +126,16 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.internal.DefaultGenerateStoresFunc()
}

// WithGenerateMetadataOnlyStoresFunc configures a custom generate metadataonly store function
func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) {
b.internal.WithGenerateMetadataOnlyStoresFunc(f)
}

// DefaultGenerateMetadataOnlyStoresFunc returns default buildMetadataOnlyStore function
func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc {
return b.internal.DefaultGenerateMetadataOnlyStoresFunc()
}

// DefaultGenerateCustomResourceStoresFunc returns default buildStores function
func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
return b.internal.DefaultGenerateCustomResourceStoresFunc()
Expand Down
11 changes: 11 additions & 0 deletions pkg/builder/types/interfaces.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/prometheus/client_golang/prometheus"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/tools/cache"

"k8s.io/kube-state-metrics/v2/pkg/customresource"
Expand All @@ -38,6 +39,7 @@ type BuilderInterface interface {
WithFieldSelectorFilter(fieldSelectors string)
WithSharding(shard int32, totalShards int)
WithContext(ctx context.Context)
WithMetadataOnlyKubeClient(c metadata.Interface)
WithKubeClient(c clientset.Interface)
WithCustomResourceClients(cs map[string]interface{})
WithUsingAPIServerCache(u bool)
Expand All @@ -46,6 +48,8 @@ type BuilderInterface interface {
WithAllowLabels(l map[string][]string) error
WithGenerateStoresFunc(f BuildStoresFunc)
DefaultGenerateStoresFunc() BuildStoresFunc
WithGenerateMetadataOnlyStoresFunc(f BuildMetadataOnlyStoresFunc)
DefaultGenerateMetadataOnlyStoresFunc() BuildMetadataOnlyStoresFunc
DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc
WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory)
Build() metricsstore.MetricsWriterList
Expand All @@ -60,6 +64,13 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator,
useAPIServerCache bool,
) []cache.Store

// BuildMetadataOnlyStoresFunc function signature that is used to return a list of cache.Store
type BuildMetadataOnlyStoresFunc func(metricFamilies []generator.FamilyGenerator,
expectedType interface{},
listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher,
useAPIServerCache bool,
) []cache.Store

// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store
type BuildCustomResourceStoresFunc func(resourceName string,
metricFamilies []generator.FamilyGenerator,
Expand Down
30 changes: 30 additions & 0 deletions pkg/util/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/discovery"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
Expand All @@ -38,6 +39,7 @@ import (

var config *rest.Config
var currentKubeClient clientset.Interface
var currentMetadataOnlyKubeClient metadata.Interface
var currentDiscoveryClient *discovery.DiscoveryClient

// CreateKubeClient creates a Kubernetes clientset and a custom resource clientset.
Expand Down Expand Up @@ -79,6 +81,34 @@ func CreateKubeClient(apiserver string, kubeconfig string) (clientset.Interface,
return kubeClient, nil
}

// CreateMetadataOnlyKubeClient creates a Kubernetes clientset and a custom resource clientset.
func CreateMetadataOnlyKubeClient(apiserver string, kubeconfig string) (metadata.Interface, error) {
if currentMetadataOnlyKubeClient != nil {
return currentMetadataOnlyKubeClient, nil
}

var err error

if config == nil {
var err error
config, err = clientcmd.BuildConfigFromFlags(apiserver, kubeconfig)
if err != nil {
return nil, err
}
}
config.UserAgent = fmt.Sprintf("%s/%s (%s/%s) kubernetes/%s", "kube-state-metrics (metadataonly)", version.Version, runtime.GOOS, runtime.GOARCH, version.Revision)
config.AcceptContentTypes = "application/vnd.kubernetes.protobuf;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1,application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1,application/json,application/vnd.kubernetes.protobuf;as=PartialObjectMetadata;g=meta.k8s.io;v=v1,application/json;as=PartialObjectMetadata;g=meta.k8s.io;v=v1"
config.ContentType = "application/vnd.kubernetes.protobuf"

kubeClient, err := metadata.NewForConfig(config)
if err != nil {
return nil, err
}

currentMetadataOnlyKubeClient = kubeClient
return kubeClient, nil
}

// CreateCustomResourceClients creates a custom resource clientset.
func CreateCustomResourceClients(apiserver string, kubeconfig string, factories ...customresource.RegistryFactory) (map[string]interface{}, error) {
// Not relying on memoized clients here because the factories are subject to change.
Expand Down

0 comments on commit cd9a7c2

Please sign in to comment.