From 3b81728582eb19a59b50fdb81349075c383cc780 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Sun, 4 Aug 2024 20:12:44 +0200 Subject: [PATCH 01/12] poc Signed-off-by: Lukasz Dziedziak --- .../pkg/dataplane/envoy/remote_bootstrap.go | 5 + go.mod | 3 - go.sum | 4 +- pkg/config/app/kuma-dp/config.go | 2 + pkg/hds/tracker/callbacks.go | 10 + pkg/util/xds/callbacks.go | 7 + pkg/xds/auth/callbacks.go | 73 ++++++- pkg/xds/bootstrap/generator.go | 1 + pkg/xds/bootstrap/parameters.go | 1 + pkg/xds/bootstrap/template_v3.go | 9 +- pkg/xds/bootstrap/types/bootstrap_request.go | 1 + pkg/xds/runtime/context.go | 2 +- .../server/callbacks/dataplane_callbacks.go | 93 ++++++++- .../callbacks/dataplane_status_tracker.go | 194 ++++++++++++++++++ pkg/xds/server/callbacks/nack_backoff.go | 22 +- pkg/xds/server/v3/components.go | 34 ++- pkg/xds/server/v3/resource_warming_forcer.go | 78 +++++++ .../meshcircuitbreaker/meshcircuitbreaker.go | 4 +- 18 files changed, 526 insertions(+), 17 deletions(-) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index a18d02eeb0fd..ccbd21fd8d6f 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -189,6 +189,11 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. }, SystemCaPath: params.SystemCaPath, } + if cfg.DataplaneRuntime.XDSConfigType == "" { + request.XDSConfigType = "sotw" + } else { + request.XDSConfigType = "delta" + } jsonBytes, err := json.MarshalIndent(request, "", " ") if err != nil { return nil, errors.Wrap(err, "could not marshal request to json") diff --git a/go.mod b/go.mod index 23f72bd8b49e..31268a6be841 100644 --- a/go.mod +++ b/go.mod @@ -226,6 +226,3 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) - -// remove once https://github.com/envoyproxy/go-control-plane/issues/875 is resolved -replace github.com/envoyproxy/go-control-plane v0.12.0 => github.com/envoyproxy/go-control-plane v0.11.2-0.20231010133108-1dfbe83bcebc diff --git a/go.sum b/go.sum index 3cf3101a654c..993cd16c0d36 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/envoyproxy/go-control-plane v0.11.2-0.20231010133108-1dfbe83bcebc h1:k6n7EmQYjNHEKr8XI3rdtFIhb0UdJSyCWnt9gvgLx5g= -github.com/envoyproxy/go-control-plane v0.11.2-0.20231010133108-1dfbe83bcebc/go.mod h1:9ODlpdyEVNyci9DZ6cdQYkwYSW1YMrnSz3xnku3cjL0= +github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= +github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index acae6eb98c3b..177c32d943f0 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -210,6 +210,8 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` + // XDSConfigType + XDSConfigType string `json:"xdsConfigType,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_type"` } type Metrics struct { diff --git a/pkg/hds/tracker/callbacks.go b/pkg/hds/tracker/callbacks.go index 8d0cba34ab45..927e13393a00 100644 --- a/pkg/hds/tracker/callbacks.go +++ b/pkg/hds/tracker/callbacks.go @@ -40,6 +40,7 @@ type tracker struct { sync.RWMutex // protects access to the fields below streamsAssociation map[xds.StreamID]core_model.ResourceKey + // deltaStreamsAssociation map[xds.StreamID]core_model.ResourceKey dpStreams map[core_model.ResourceKey]streams } @@ -56,6 +57,7 @@ func NewCallbacks( return &tracker{ resourceManager: resourceManager, streamsAssociation: map[xds.StreamID]core_model.ResourceKey{}, + // deltaStreamsAssociation: map[xds.StreamID]core_model.ResourceKey{}, dpStreams: map[core_model.ResourceKey]streams{}, config: config, log: log, @@ -69,6 +71,14 @@ func NewCallbacks( } } +func (t *tracker) OnDeltaStreamOpen(ctx context.Context, streamID int64) error { + return t.OnStreamOpen(ctx, streamID) +} + +func (t *tracker) OnDeltaStreamClosed(streamID xds.StreamID) { + t.OnStreamClosed(streamID) +} + func (t *tracker) OnStreamOpen(ctx context.Context, streamID int64) error { t.metrics.StreamsActiveInc() return nil diff --git a/pkg/util/xds/callbacks.go b/pkg/util/xds/callbacks.go index 95bbb5a55fb3..f3914366409f 100644 --- a/pkg/util/xds/callbacks.go +++ b/pkg/util/xds/callbacks.go @@ -94,4 +94,11 @@ type RestCallbacks interface { type MultiCallbacks interface { Callbacks RestCallbacks + DeltaCallbacks } + +type MultiXDSCallbacks interface { + Callbacks + DeltaCallbacks +} + diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 4578111a4e23..927e6b856839 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -10,6 +10,7 @@ import ( "google.golang.org/grpc/metadata" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/core" core_mesh "github.com/kumahq/kuma/pkg/core/resources/apis/mesh" core_manager "github.com/kumahq/kuma/pkg/core/resources/manager" "github.com/kumahq/kuma/pkg/core/resources/model" @@ -30,7 +31,7 @@ type DPNotFoundRetry struct { MaxTimes uint } -func NewCallbacks(resManager core_manager.ReadOnlyResourceManager, authenticator Authenticator, dpNotFoundRetry DPNotFoundRetry) util_xds.Callbacks { +func NewCallbacks(resManager core_manager.ReadOnlyResourceManager, authenticator Authenticator, dpNotFoundRetry DPNotFoundRetry) util_xds.MultiXDSCallbacks { if dpNotFoundRetry.Backoff == 0 { // backoff cannot be 0 dpNotFoundRetry.Backoff = 1 * time.Millisecond } @@ -38,6 +39,7 @@ func NewCallbacks(resManager core_manager.ReadOnlyResourceManager, authenticator resManager: resManager, authenticator: authenticator, streams: map[core_xds.StreamID]stream{}, + deltaStreams: map[core_xds.StreamID]stream{}, dpNotFoundRetry: dpNotFoundRetry, } } @@ -51,6 +53,7 @@ type authCallbacks struct { sync.RWMutex // protects streams streams map[core_xds.StreamID]stream + deltaStreams map[core_xds.StreamID]stream } type stream struct { @@ -62,7 +65,7 @@ type stream struct { nodeID string } -var _ util_xds.Callbacks = &authCallbacks{} +var _ util_xds.MultiXDSCallbacks = &authCallbacks{} func (a *authCallbacks) OnStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { a.Lock() @@ -100,6 +103,72 @@ func (a *authCallbacks) OnStreamRequest(streamID core_xds.StreamID, req util_xds return nil } +func (a *authCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { + a.Lock() + defer a.Unlock() + + a.deltaStreams[streamID] = stream{ + ctx: ctx, + resource: nil, + } + + core.Log.V(1).Info("OnDeltaStreamOpen", "streamID", streamID) + return nil +} + +func (a *authCallbacks) OnDeltaStreamClosed(streamID int64) { + a.Lock() + delete(a.deltaStreams, streamID) + a.Unlock() +} + +func (a *authCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, req util_xds.DeltaDiscoveryRequest) error { + s, err := a.deltaStream(streamID, req) + if err != nil { + return err + } + core.Log.V(1).Info("OnStreamDeltaRequest auth", "req", req) + + credential, err := ExtractCredential(s.ctx) + if err != nil { + return errors.Wrap(err, "could not extract credential from DiscoveryRequest") + } + if err := a.authenticator.Authenticate(user.Ctx(s.ctx, user.ControlPlane), s.resource, credential); err != nil { + return errors.Wrap(err, "authentication failed") + } + a.Lock() + a.deltaStreams[streamID] = s + a.Unlock() + return nil +} + +func (a *authCallbacks) deltaStream(streamID core_xds.StreamID, req util_xds.DeltaDiscoveryRequest) (stream, error) { + a.RLock() + s, ok := a.deltaStreams[streamID] + a.RUnlock() + if !ok { + return stream{}, errors.New("stream is not present") + } + + if s.nodeID == "" { + s.nodeID = req.NodeId() + } + + if s.nodeID != req.NodeId() { + return stream{}, errors.Errorf("stream was authenticated for ID %s. Received request is for node with ID %s. Node ID cannot be changed after stream is initialized", s.nodeID, req.NodeId()) + } + + if s.resource == nil { + md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) + res, err := a.resource(user.Ctx(s.ctx, user.ControlPlane), md, req.NodeId()) + if err != nil { + return stream{}, err + } + s.resource = res + } + return s, nil +} + func (a *authCallbacks) stream(streamID core_xds.StreamID, req util_xds.DiscoveryRequest) (stream, error) { a.RLock() s, ok := a.streams[streamID] diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index def46fb59a35..1af4a2472c12 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -113,6 +113,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr MetricsCertPath: request.MetricsResources.CertPath, MetricsKeyPath: request.MetricsResources.KeyPath, SystemCaPath: request.SystemCaPath, + UseDelta: request.XDSConfigType == "delta", } setAdminPort := func(adminPortFromResource uint32) { diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index c6a8fe67d93e..71a5616ef2f3 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -50,4 +50,5 @@ type configParameters struct { IsGatewayDataplane bool Resources types.ProxyResources SystemCaPath string + UseDelta bool } diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go index 523873d8d5d9..3f14ab8d9ce3 100644 --- a/pkg/xds/bootstrap/template_v3.go +++ b/pkg/xds/bootstrap/template_v3.go @@ -106,6 +106,11 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada }, }) } + configType := envoy_core_v3.ApiConfigSource_DELTA_GRPC + // if parameters.UseDelta { + // configType = envoy_core_v3.ApiConfigSource_DELTA_GRPC + // } + res := &envoy_bootstrap_v3.Bootstrap{ Node: &envoy_core_v3.Node{ Id: parameters.Id, @@ -168,7 +173,7 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada ResourceApiVersion: envoy_core_v3.ApiVersion_V3, }, AdsConfig: &envoy_core_v3.ApiConfigSource{ - ApiType: envoy_core_v3.ApiConfigSource_GRPC, + ApiType: configType, TransportApiVersion: envoy_core_v3.ApiVersion_V3, SetNodeOnFirstMessageOnly: true, GrpcServices: []*envoy_core_v3.GrpcService{ @@ -228,7 +233,7 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada } if parameters.HdsEnabled { res.HdsConfig = &envoy_core_v3.ApiConfigSource{ - ApiType: envoy_core_v3.ApiConfigSource_GRPC, + ApiType: configType, TransportApiVersion: envoy_core_v3.ApiVersion_V3, SetNodeOnFirstMessageOnly: true, GrpcServices: []*envoy_core_v3.GrpcService{ diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index 89d5521f172e..7c77390491bc 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -19,6 +19,7 @@ type BootstrapRequest struct { Workdir string `json:"workdir"` MetricsResources MetricsResources `json:"metricsResources"` SystemCaPath string `json:"systemCaPath"` + XDSConfigType string `json:"xdsConfigType,omitempty"` } type Version struct { diff --git a/pkg/xds/runtime/context.go b/pkg/xds/runtime/context.go index 522655322753..0571e1133de6 100644 --- a/pkg/xds/runtime/context.go +++ b/pkg/xds/runtime/context.go @@ -17,7 +17,7 @@ type XDSRuntimeContext struct { DpProxyAuthenticator xds_auth.Authenticator ZoneProxyAuthenticator xds_auth.Authenticator Hooks *xds_hooks.Hooks - ServerCallbacks util_xds.Callbacks + ServerCallbacks util_xds.MultiXDSCallbacks Metrics *xds_metrics.Metrics } diff --git a/pkg/xds/server/callbacks/dataplane_callbacks.go b/pkg/xds/server/callbacks/dataplane_callbacks.go index f0c55a686ef1..e9d1569c34a1 100644 --- a/pkg/xds/server/callbacks/dataplane_callbacks.go +++ b/pkg/xds/server/callbacks/dataplane_callbacks.go @@ -35,13 +35,15 @@ type xdsCallbacks struct { sync.RWMutex dpStreams map[core_xds.StreamID]dpStream + dpDeltaStreams map[core_xds.StreamID]dpStream activeStreams map[core_model.ResourceKey]int } -func DataplaneCallbacksToXdsCallbacks(callbacks DataplaneCallbacks) util_xds.Callbacks { +func DataplaneCallbacksToXdsCallbacks(callbacks DataplaneCallbacks) util_xds.MultiXDSCallbacks { return &xdsCallbacks{ callbacks: callbacks, dpStreams: map[core_xds.StreamID]dpStream{}, + dpDeltaStreams: map[core_xds.StreamID]dpStream{}, activeStreams: map[core_model.ResourceKey]int{}, } } @@ -51,7 +53,8 @@ type dpStream struct { ctx context.Context } -var _ util_xds.Callbacks = &xdsCallbacks{} +var _ util_xds.MultiXDSCallbacks = &xdsCallbacks{} + func (d *xdsCallbacks) OnStreamClosed(streamID core_xds.StreamID) { var lastStreamDpKey *core_model.ResourceKey @@ -129,6 +132,92 @@ func (d *xdsCallbacks) OnStreamRequest(streamID core_xds.StreamID, request util_ return nil } +func (d *xdsCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { + d.Lock() + defer d.Unlock() + dps := dpStream{ + ctx: ctx, + } + d.dpDeltaStreams[streamID] = dps + return nil +} + +func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { + var lastStreamDpKey *core_model.ResourceKey + d.Lock() + dpStream := d.dpDeltaStreams[streamID] + if dpKey := dpStream.dp; dpKey != nil { + d.activeStreams[*dpKey]-- + if d.activeStreams[*dpKey] == 0 { + lastStreamDpKey = dpKey + delete(d.activeStreams, *dpKey) + } + } + delete(d.dpDeltaStreams, streamID) + d.Unlock() + if lastStreamDpKey != nil { + // execute callback after lock is freed, so heavy callback implementation won't block every callback for every DPP. + d.callbacks.OnProxyDisconnected(dpStream.ctx, streamID, *lastStreamDpKey) + } +} + +func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request util_xds.DeltaDiscoveryRequest) error { + if request.NodeId() == "" { + // from https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-versioning: + // Only the first request on a stream is guaranteed to carry the node identifier. + // The subsequent discovery requests on the same stream may carry an empty node identifier. + // This holds true regardless of the acceptance of the discovery responses on the same stream. + // The node identifier should always be identical if present more than once on the stream. + // It is sufficient to only check the first message for the node identifier as a result. + return nil + } + + d.RLock() + alreadyProcessed := d.dpDeltaStreams[streamID].dp != nil + d.RUnlock() + if alreadyProcessed { + return nil + } + + proxyId, err := core_xds.ParseProxyIdFromString(request.NodeId()) + if err != nil { + return errors.Wrap(err, "invalid node ID") + } + dpKey := proxyId.ToResourceKey() + metadata := core_xds.DataplaneMetadataFromXdsMetadata(request.Metadata()) + if metadata == nil { + return errors.New("metadata in xDS Node cannot be nil") + } + + d.Lock() + // in case client will open 2 concurrent request for the same streamID then + // we don't to increment the counter twice, so checking once again that stream + // wasn't processed + alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil + if alreadyProcessed { + return nil + } + + dpStream := d.dpDeltaStreams[streamID] + dpStream.dp = &dpKey + d.dpDeltaStreams[streamID] = dpStream + + activeStreams := d.activeStreams[dpKey] + d.activeStreams[dpKey]++ + d.Unlock() + + if activeStreams == 0 { + if err := d.callbacks.OnProxyConnected(streamID, dpKey, dpStream.ctx, *metadata); err != nil { + return err + } + } else { + if err := d.callbacks.OnProxyReconnected(streamID, dpKey, dpStream.ctx, *metadata); err != nil { + return err + } + } + return nil +} + func (d *xdsCallbacks) OnStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { d.Lock() defer d.Unlock() diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index 8df8912364a7..2e606cd2b284 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -21,6 +21,7 @@ var statusTrackerLog = core.Log.WithName("xds").WithName("status-tracker") type DataplaneStatusTracker interface { util_xds.Callbacks + util_xds.DeltaCallbacks GetStatusAccessor(streamID int64) (SubscriptionStatusAccessor, bool) } @@ -38,6 +39,7 @@ func NewDataplaneStatusTracker( runtimeInfo: runtimeInfo, createStatusSink: createStatusSink, streams: make(map[int64]*streamState), + deltaStreams: make(map[int64]*streamState), } } @@ -49,6 +51,7 @@ type dataplaneStatusTracker struct { createStatusSink DataplaneInsightSinkFactoryFunc mu sync.RWMutex // protects access to the fields below streams map[int64]*streamState + deltaStreams map[int64]*streamState } type streamState struct { @@ -275,3 +278,194 @@ func (s *streamState) GetStatus() (core_model.ResourceKey, *mesh_proto.Discovery func (s *streamState) Close() { close(s.stop) } + +// OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS). +// Returning an error will end processing and close the stream. OnStreamClosed will still be called. +func (c *dataplaneStatusTracker) OnDeltaStreamOpen(ctx context.Context, streamID int64, typ string) error { + c.mu.Lock() // write access to the map of all ADS streams + defer c.mu.Unlock() + + // initialize subscription + now := core.Now() + subscription := &mesh_proto.DiscoverySubscription{ + Id: core.NewUUID(), + ControlPlaneInstanceId: c.runtimeInfo.GetInstanceId(), + ConnectTime: util_proto.MustTimestampProto(now), + Status: mesh_proto.NewSubscriptionStatus(now), + Version: mesh_proto.NewVersion(), + } + // initialize state per ADS stream + state := &streamState{ + stop: make(chan struct{}), + subscription: subscription, + } + // save + c.deltaStreams[streamID] = state + + statusTrackerLog.V(1).Info("proxy connecting", "streamID", streamID, "type", typ, "subscriptionID", subscription.Id) + return nil +} + +// OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID. +func (c *dataplaneStatusTracker) OnDeltaStreamClosed(streamID int64) { + c.mu.Lock() // write access to the map of all ADS streams + defer c.mu.Unlock() + + state := c.deltaStreams[streamID] + if state == nil { + statusTrackerLog.Info("[WARNING] proxy disconnected but no state in the status_tracker", "streamID", streamID) + return + } + + delete(c.deltaStreams, streamID) + + // finilize subscription + state.mu.Lock() // write access to the per Dataplane info + subscription := state.subscription + subscription.DisconnectTime = util_proto.MustTimestampProto(core.Now()) + state.mu.Unlock() + + // trigger final flush + state.Close() + + log := statusTrackerLog.WithValues( + "streamID", streamID, + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "subscriptionID", state.subscription.Id, + ) + + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues("subscription", subscription) + } + + log.Info("proxy disconnected") +} + +// OnStreamRequest is called once a request is received on a stream. +// Returning an error will end processing and close the stream. OnStreamClosed will still be called. +func (c *dataplaneStatusTracker) OnStreamDeltaRequest(streamID int64, req util_xds.DeltaDiscoveryRequest) error { + c.mu.RLock() // read access to the map of all ADS streams + defer c.mu.RUnlock() + + state := c.deltaStreams[streamID] + + state.mu.Lock() // write access to the per Dataplane info + defer state.mu.Unlock() + + if state.dataplaneId == (core_model.ResourceKey{}) { + // Infer the Dataplane ID. + if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { + state.dataplaneId = proxyId.ToResourceKey() + var dpType core_model.ResourceType + md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) + + // If the dataplane was started with a resource YAML, then it + // will be serialized in the node metadata and we would know + // the underlying type directly. Since that is optional, we + // can't depend on it here, so we map from the proxy type, + // which is guaranteed. + switch md.GetProxyType() { + case mesh_proto.IngressProxyType: + dpType = core_mesh.ZoneIngressType + case mesh_proto.DataplaneProxyType: + dpType = core_mesh.DataplaneType + case mesh_proto.EgressProxyType: + dpType = core_mesh.ZoneEgressType + } + + log := statusTrackerLog.WithValues( + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "streamID", streamID, + "type", md.GetProxyType(), + "dpVersion", md.GetVersion().GetKumaDp().GetVersion(), + "subscriptionID", state.subscription.Id, + ) + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues("node", req.Node()) + } + log.Info("proxy connected") + + if md.GetVersion() != nil { + state.subscription.Version = md.GetVersion() + } else { + statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) + } + // Kick off the async Dataplane status flusher. + go c.createStatusSink(dpType, state).Start(state.stop) + } else { + statusTrackerLog.Error(err, "failed to parse Dataplane Id out of DiscoveryRequest", "streamid", streamID, "req", req) + } + } + + subscription := state.subscription + log := statusTrackerLog.WithValues( + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "streamID", streamID, + "type", shortEnvoyType(req.GetTypeUrl()), + "resourceVersion", req.GetInitialResourceVersions(), + ) + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues( + "resourceNames", req.GetResourceNamesSubscribe(), + "subscriptionID", subscription.Id, + "nonce", req.GetResponseNonce(), + ) + } + + // update Dataplane status + if req.GetResponseNonce() != "" { + subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) + if req.HasErrors() { + log.Info("config rejected") + subscription.Status.Total.ResponsesRejected++ + subscription.Status.StatsOf(req.GetTypeUrl()).ResponsesRejected++ + } else { + log.V(1).Info("config accepted") + subscription.Status.Total.ResponsesAcknowledged++ + subscription.Status.StatsOf(req.GetTypeUrl()).ResponsesAcknowledged++ + } + } else { + if !statusTrackerLog.V(1).Enabled() { // it was already added, no need to add it twice + log = log.WithValues("resourceNames", req.GetResourceNamesSubscribe()) + } + log.Info("config requested") + } + return nil +} + +// OnStreamResponse is called immediately prior to sending a response on a stream. +func (c *dataplaneStatusTracker) OnStreamDeltaResponse(streamID int64, req util_xds.DeltaDiscoveryRequest, resp util_xds.DeltaDiscoveryResponse) { + c.mu.RLock() // read access to the map of all ADS streams + defer c.mu.RUnlock() + + state := c.deltaStreams[streamID] + + state.mu.Lock() // write access to the per Dataplane info + defer state.mu.Unlock() + + // update Dataplane status + subscription := state.subscription + subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) + subscription.Status.Total.ResponsesSent++ + subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ + + log := statusTrackerLog.WithValues( + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "streamID", streamID, + "type", shortEnvoyType(req.GetTypeUrl()), + "requestedResourceNames", req.GetResourceNamesSubscribe(), + "resourceCount", len(resp.GetResources()), + ) + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues( + "subscriptionID", subscription.Id, + "nonce", resp.GetNonce(), + ) + } + + log.V(1).Info("config sent") +} diff --git a/pkg/xds/server/callbacks/nack_backoff.go b/pkg/xds/server/callbacks/nack_backoff.go index b5236da81390..fe98ef62ced6 100644 --- a/pkg/xds/server/callbacks/nack_backoff.go +++ b/pkg/xds/server/callbacks/nack_backoff.go @@ -14,9 +14,9 @@ type nackBackoff struct { util_xds.NoopCallbacks } -var _ util_xds.Callbacks = &nackBackoff{} +var _ util_xds.MultiXDSCallbacks = &nackBackoff{} -func NewNackBackoff(backoff time.Duration) util_xds.Callbacks { +func NewNackBackoff(backoff time.Duration) util_xds.MultiXDSCallbacks { return &nackBackoff{ backoff: backoff, } @@ -38,3 +38,21 @@ func (n *nackBackoff) OnStreamResponse(_ int64, request util_xds.DiscoveryReques time.Sleep(n.backoff) } } + +func (n *nackBackoff) OnStreamDeltaResponse(_ int64, request util_xds.DeltaDiscoveryRequest, _ util_xds.DeltaDiscoveryResponse) { + if request.HasErrors() { + // When DiscoveryRequest contains errors, it means that Envoy rejected configuration generated by Control Plane + // It may happen for several reasons: + // 1) Eventual consistency - ex. listener consists reference to cluster which does not exist because listener was send before cluster (there is no ordering of responses) + // 2) Config is valid from CP side but invalid from Envoy side - ex. something already listening at this address:port + // + // Second case is especially dangerous because we will end up in a loop. + // CP is constantly trying to send a config and Envoy immediately rejects the config. + // Without this backoff, CP is under a lot of pressure from faulty Envoy. + // + // It is safe to sleep here because OnStreamResponse is executed in the goroutine of a single ADS stream + nackLog.Info("config was previously rejected by Envoy. Applying backoff before resending it", "backoff", n.backoff, "nodeID", request.NodeId(), "reason", request.ErrorMsg()) + time.Sleep(n.backoff) + } +} + diff --git a/pkg/xds/server/v3/components.go b/pkg/xds/server/v3/components.go index 55cb870dd8c4..7ec520980bf7 100644 --- a/pkg/xds/server/v3/components.go +++ b/pkg/xds/server/v3/components.go @@ -5,8 +5,13 @@ import ( "time" envoy_service_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/envoyproxy/go-control-plane/pkg/server/config" envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v3" + envoy_server_delta "github.com/envoyproxy/go-control-plane/pkg/server/delta/v3" + envoy_server_rest "github.com/envoyproxy/go-control-plane/pkg/server/rest/v3" + envoy_server_sotw "github.com/envoyproxy/go-control-plane/pkg/server/sotw/v3" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" core_model "github.com/kumahq/kuma/pkg/core/resources/model" @@ -65,10 +70,35 @@ func RegisterXDS( callbacks = append(callbacks, util_xds_v3.AdaptCallbacks(cb)) } - srv := envoy_server.NewServer(context.Background(), xdsContext.Cache(), callbacks) + deltaCallbacks := util_xds_v3.CallbacksChain{ + util_xds_v3.NewControlPlaneIdCallbacks(rt.GetInstanceId()), + util_xds_v3.AdaptDeltaCallbacks(statsCallbacks), + util_xds_v3.AdaptDeltaCallbacks(authCallbacks), + util_xds_v3.AdaptDeltaCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(metadataTracker)), + util_xds_v3.AdaptDeltaCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks(xds_callbacks.NewDataplaneSyncTracker(watchdogFactory.New))), + util_xds_v3.AdaptDeltaCallbacks(xds_callbacks.DataplaneCallbacksToXdsCallbacks( + xds_callbacks.NewDataplaneLifecycle(rt.AppContext(), rt.ResourceManager(), authenticator, rt.Config().XdsServer.DataplaneDeregistrationDelay.Duration, rt.GetInstanceId())), + ), + util_xds_v3.AdaptDeltaCallbacks(DefaultDataplaneStatusTracker(rt, envoyCpCtx.Secrets)), + util_xds_v3.AdaptDeltaCallbacks(xds_callbacks.NewNackBackoff(rt.Config().XdsServer.NACKBackoff.Duration)), + newResourceWarmingForcer(xdsContext.Cache(), xdsContext.Hasher()), + } + + if cb := rt.XDS().ServerCallbacks; cb != nil { + deltaCallbacks = append(deltaCallbacks, util_xds_v3.AdaptDeltaCallbacks(cb)) + } + + + rest := envoy_server_rest.NewServer(xdsContext.Cache(), callbacks) + sotw := envoy_server_sotw.NewServer(context.Background(), xdsContext.Cache(), callbacks) + ordered := func(o *config.Opts) { + o.Ordered = true + } + delta := envoy_server_delta.NewServer(context.Background(), xdsContext.Cache(), deltaCallbacks, ordered) + newServerAdvanced := envoy_server.NewServerAdvanced(rest, sotw, delta) xdsServerLog.Info("registering Aggregated Discovery Service V3 in Dataplane Server") - envoy_service_discovery.RegisterAggregatedDiscoveryServiceServer(rt.DpServer().GrpcServer(), srv) + envoy_service_discovery.RegisterAggregatedDiscoveryServiceServer(rt.DpServer().GrpcServer(), newServerAdvanced) return nil } diff --git a/pkg/xds/server/v3/resource_warming_forcer.go b/pkg/xds/server/v3/resource_warming_forcer.go index 5064420faf53..ca5821baeced 100644 --- a/pkg/xds/server/v3/resource_warming_forcer.go +++ b/pkg/xds/server/v3/resource_warming_forcer.go @@ -84,6 +84,13 @@ func (r *resourceWarmingForcer) OnStreamClosed(streamID int64, _ *envoy_core.Nod delete(r.nodeIDs, streamID) } +func (r *resourceWarmingForcer) OnDeltaStreamClosed(streamID int64, _ *envoy_core.Node) { + r.Lock() + defer r.Unlock() + delete(r.lastEndpointNonces, streamID) + delete(r.nodeIDs, streamID) +} + func (r *resourceWarmingForcer) OnStreamRequest(streamID xds.StreamID, request *envoy_sd.DiscoveryRequest) error { if request.TypeUrl != envoy_resource.EndpointType { return nil // we force Cluster warming only on receiving the same EDS Discovery Request @@ -114,6 +121,57 @@ func (r *resourceWarmingForcer) OnStreamRequest(streamID xds.StreamID, request * return nil } +func (r *resourceWarmingForcer) OnStreamDeltaRequest(streamID xds.StreamID, request *envoy_sd.DeltaDiscoveryRequest) error { + warmingForcerLog.Info("check something", "streamdi", streamID, "request.TypeUrl", request.TypeUrl, "request.ResponseNonce", request.ResponseNonce) + if request.TypeUrl != envoy_resource.EndpointType { + return nil // we force Cluster warming only on receiving the same EDS Discovery Request + } + if request.ResponseNonce == "" { + return nil // initial request, no need to force warming + } + if request.ErrorDetail != nil { + return nil // we only care about ACKs, otherwise we can get 2 Nonces with multiple NACKs + } + warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") + r.Lock() + lastEndpointNonce := r.lastEndpointNonces[streamID] + r.lastEndpointNonces[streamID] = request.ResponseNonce + nodeID := r.nodeIDs[streamID] + if nodeID == "" { + nodeID = r.hasher.ID(request.Node) // request.Node can be set only on first request therefore we need to save it + r.nodeIDs[streamID] = nodeID + } + r.Unlock() + warmingForcerLog.Info("HMMM", "lastEndpointNonce", lastEndpointNonce, "request.ResponseNonce ", request.ResponseNonce ) + if lastEndpointNonce == request.ResponseNonce || lastEndpointNonce == "" { + warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") + if err := r.deltaForceNewEndpointsVersion(nodeID); err != nil { + warmingForcerLog.Error(err, "could not force cluster warming") + } + } + return nil +} + +func (r *resourceWarmingForcer) OnStreamDeltaResponse(streamID int64, request *envoy_sd.DeltaDiscoveryRequest, _ *envoy_sd.DeltaDiscoveryResponse) { + if request.TypeUrl != envoy_resource.ClusterType { + return + } + r.Lock() + + nodeID := r.nodeIDs[streamID] + if nodeID == "" { + nodeID = r.hasher.ID(request.Node) // request.Node can be set only on first request therefore we need to save it + r.nodeIDs[streamID] = nodeID + } + r.Unlock() + if request.TypeUrl == envoy_resource.ClusterType { + warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") + if err := r.deltaForceNewEndpointsVersion(nodeID); err != nil { + warmingForcerLog.Error(err, "could not force cluster warming") + } + } +} + func (r *resourceWarmingForcer) forceNewEndpointsVersion(nodeID string) error { snapshot, err := r.cache.GetSnapshot(nodeID) if err != nil { @@ -132,4 +190,24 @@ func (r *resourceWarmingForcer) forceNewEndpointsVersion(nodeID string) error { return nil } +func (r *resourceWarmingForcer) deltaForceNewEndpointsVersion(nodeID string) error { + snapshot, err := r.cache.GetSnapshot(nodeID) + if err != nil { + return nil // GetSnapshot returns an error if there is no snapshot. We don't need to force on a new snapshot + } + cacheSnapshot, ok := snapshot.(*envoy_cache.Snapshot) + if !ok { + return errors.New("couldn't convert snapshot from cache to envoy Snapshot") + } + responseType, err := envoy_cache.GetResponseTypeURL(types.Endpoint) + if err != nil { + return err + } + for _, endpoint := range cacheSnapshot.VersionMap[responseType]{ + cacheSnapshot.VersionMap[responseType][endpoint] = "" + } + + return nil +} + var _ envoy_xds.Callbacks = &resourceWarmingForcer{} diff --git a/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go b/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go index 455c5895251f..801b2fac3913 100644 --- a/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go +++ b/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go @@ -2,6 +2,7 @@ package meshcircuitbreaker import ( "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -81,6 +82,7 @@ func MeshCircuitBreaker() { // when Expect(kubernetes.Cluster.Install(YamlK8s(config))).To(Succeed()) + time.Sleep(1*time.Hour) // then Eventually(func(g Gomega) ([]client.FailureResponse, error) { return client.CollectResponsesAndFailures( @@ -98,7 +100,7 @@ func MeshCircuitBreaker() { ContainElement(HaveField("ResponseCode", 503)), )) }, - Entry("outbound circuit breaker", fmt.Sprintf(` + FEntry("outbound circuit breaker", fmt.Sprintf(` apiVersion: kuma.io/v1alpha1 kind: MeshCircuitBreaker metadata: From a1bb87547e1b59aa44d6f1556ebcdf5dc0532507 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 3 Sep 2024 08:48:09 -0500 Subject: [PATCH 02/12] cleanup Signed-off-by: Lukasz Dziedziak --- pkg/hds/tracker/callbacks.go | 10 +- pkg/util/xds/callbacks.go | 41 ++- pkg/util/xds/v3/callbacks.go | 29 ++ pkg/xds/auth/callbacks.go | 68 ++-- pkg/xds/bootstrap/parameters.go | 2 +- pkg/xds/envoy/imports.go | 27 ++ pkg/xds/server/callbacks/1 | 31 ++ pkg/xds/server/callbacks/2 | 30 ++ .../server/callbacks/dataplane_callbacks.go | 155 ++++----- .../callbacks/dataplane_status_tracker.go | 305 +++++------------- pkg/xds/server/callbacks/nack_backoff.go | 20 +- pkg/xds/server/v3/components.go | 5 +- pkg/xds/server/v3/resource_warming_forcer.go | 78 ----- .../meshcircuitbreaker/meshcircuitbreaker.go | 4 +- 14 files changed, 319 insertions(+), 486 deletions(-) create mode 100644 pkg/xds/server/callbacks/1 create mode 100644 pkg/xds/server/callbacks/2 diff --git a/pkg/hds/tracker/callbacks.go b/pkg/hds/tracker/callbacks.go index 927e13393a00..1b483b97277a 100644 --- a/pkg/hds/tracker/callbacks.go +++ b/pkg/hds/tracker/callbacks.go @@ -41,7 +41,7 @@ type tracker struct { sync.RWMutex // protects access to the fields below streamsAssociation map[xds.StreamID]core_model.ResourceKey // deltaStreamsAssociation map[xds.StreamID]core_model.ResourceKey - dpStreams map[core_model.ResourceKey]streams + dpStreams map[core_model.ResourceKey]streams } func NewCallbacks( @@ -58,10 +58,10 @@ func NewCallbacks( resourceManager: resourceManager, streamsAssociation: map[xds.StreamID]core_model.ResourceKey{}, // deltaStreamsAssociation: map[xds.StreamID]core_model.ResourceKey{}, - dpStreams: map[core_model.ResourceKey]streams{}, - config: config, - log: log, - metrics: metrics, + dpStreams: map[core_model.ResourceKey]streams{}, + config: config, + log: log, + metrics: metrics, reconciler: &reconciler{ cache: cache, hasher: hasher, diff --git a/pkg/util/xds/callbacks.go b/pkg/util/xds/callbacks.go index f3914366409f..4a40b900ce93 100644 --- a/pkg/util/xds/callbacks.go +++ b/pkg/util/xds/callbacks.go @@ -3,52 +3,50 @@ package xds import ( "context" - discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" ) -// DiscoveryRequest defines interface over real Envoy's DiscoveryRequest. -type DiscoveryRequest interface { +type Request interface { NodeId() string - // Node returns either a v2 or v3 Node Node() interface{} Metadata() *structpb.Struct - VersionInfo() string - GetTypeUrl() string GetResponseNonce() string - GetResourceNames() []string + GetTypeUrl() string HasErrors() bool ErrorMsg() string + VersionInfo() string + GetResourceNames() []string } -// DiscoveryResponse defines interface over real Envoy's DiscoveryResponse. -type DiscoveryResponse interface { +type Response interface { GetTypeUrl() string - VersionInfo() string GetResources() []*anypb.Any GetNonce() string + VersionInfo() string + GetNumberOfResources() int +} + +// DiscoveryRequest defines interface over real Envoy's DiscoveryRequest. +type DiscoveryRequest interface { + Request +} + +// DiscoveryResponse defines interface over real Envoy's DiscoveryResponse. +type DiscoveryResponse interface { + Response } type DeltaDiscoveryRequest interface { - NodeId() string - // Node returns either a v2 or v3 Node - Node() interface{} - Metadata() *structpb.Struct - GetTypeUrl() string - GetResponseNonce() string + Request GetResourceNamesSubscribe() []string GetInitialResourceVersions() map[string]string - HasErrors() bool - ErrorMsg() string } // DeltaDiscoveryResponse defines interface over real Envoy's DeltaDiscoveryResponse. type DeltaDiscoveryResponse interface { - GetTypeUrl() string - GetResources() []*discoveryv3.Resource + Response GetRemovedResources() []string - GetNonce() string } // Callbacks defines Callbacks for xDS streaming requests. The difference over real go-control-plane Callbacks is that it takes an DiscoveryRequest / DiscoveryResponse interface. @@ -101,4 +99,3 @@ type MultiXDSCallbacks interface { Callbacks DeltaCallbacks } - diff --git a/pkg/util/xds/v3/callbacks.go b/pkg/util/xds/v3/callbacks.go index 861b1321866b..95b535ff422a 100644 --- a/pkg/util/xds/v3/callbacks.go +++ b/pkg/util/xds/v3/callbacks.go @@ -6,6 +6,7 @@ import ( envoy_core "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_sd "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" envoy_xds "github.com/envoyproxy/go-control-plane/pkg/server/v3" + "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/structpb" "github.com/kumahq/kuma/pkg/util/xds" @@ -175,6 +176,10 @@ type discoveryResponse struct { *envoy_sd.DiscoveryResponse } +func (d *discoveryResponse) GetNumberOfResources() int { + return len(d.Resources) +} + func (d *discoveryResponse) VersionInfo() string { return d.GetVersionInfo() } @@ -211,12 +216,36 @@ func (d *deltaDiscoveryRequest) GetInitialResourceVersions() map[string]string { return d.InitialResourceVersions } +func (d *deltaDiscoveryRequest) GetResourceNames() []string { + return d.GetResourceNamesSubscribe() +} + +func (d *deltaDiscoveryRequest) VersionInfo() string { + return "" +} + var _ xds.DeltaDiscoveryRequest = &deltaDiscoveryRequest{} type deltaDiscoveryResponse struct { *envoy_sd.DeltaDiscoveryResponse } +func (d *deltaDiscoveryResponse) VersionInfo() string { + return d.SystemVersionInfo +} + +func (d *deltaDiscoveryResponse) GetResources() []*anypb.Any { + resources := []*anypb.Any{} + for _, res := range d.Resources { + resources = append(resources, res.Resource) + } + return resources +} + +func (d *deltaDiscoveryResponse) GetNumberOfResources() int { + return len(d.Resources) +} + var _ xds.DeltaDiscoveryResponse = &deltaDiscoveryResponse{} func (d *deltaDiscoveryResponse) GetTypeUrl() string { diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 927e6b856839..2666310567ee 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -39,7 +39,7 @@ func NewCallbacks(resManager core_manager.ReadOnlyResourceManager, authenticator resManager: resManager, authenticator: authenticator, streams: map[core_xds.StreamID]stream{}, - deltaStreams: map[core_xds.StreamID]stream{}, + deltaStreams: map[core_xds.StreamID]stream{}, dpNotFoundRetry: dpNotFoundRetry, } } @@ -53,7 +53,7 @@ type authCallbacks struct { sync.RWMutex // protects streams streams map[core_xds.StreamID]stream - deltaStreams map[core_xds.StreamID]stream + deltaStreams map[core_xds.StreamID]stream } type stream struct { @@ -85,22 +85,7 @@ func (a *authCallbacks) OnStreamClosed(streamID core_xds.StreamID) { } func (a *authCallbacks) OnStreamRequest(streamID core_xds.StreamID, req util_xds.DiscoveryRequest) error { - s, err := a.stream(streamID, req) - if err != nil { - return err - } - - credential, err := ExtractCredential(s.ctx) - if err != nil { - return errors.Wrap(err, "could not extract credential from DiscoveryRequest") - } - if err := a.authenticator.Authenticate(user.Ctx(s.ctx, user.ControlPlane), s.resource, credential); err != nil { - return errors.Wrap(err, "authentication failed") - } - a.Lock() - a.streams[streamID] = s - a.Unlock() - return nil + return a.onStreamRequest(streamID, req, false) } func (a *authCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { @@ -123,7 +108,11 @@ func (a *authCallbacks) OnDeltaStreamClosed(streamID int64) { } func (a *authCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, req util_xds.DeltaDiscoveryRequest) error { - s, err := a.deltaStream(streamID, req) + return a.onStreamRequest(streamID, req, true) +} + +func (a *authCallbacks) onStreamRequest(streamID core_xds.StreamID, req util_xds.Request, isDelta bool) error { + s, err := a.stream(streamID, req, isDelta) if err != nil { return err } @@ -137,41 +126,24 @@ func (a *authCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, req uti return errors.Wrap(err, "authentication failed") } a.Lock() - a.deltaStreams[streamID] = s + if isDelta { + a.deltaStreams[streamID] = s + } else { + a.streams[streamID] = s + } a.Unlock() return nil } -func (a *authCallbacks) deltaStream(streamID core_xds.StreamID, req util_xds.DeltaDiscoveryRequest) (stream, error) { +func (a *authCallbacks) stream(streamID core_xds.StreamID, req util_xds.Request, isDelta bool) (stream, error) { a.RLock() - s, ok := a.deltaStreams[streamID] - a.RUnlock() - if !ok { - return stream{}, errors.New("stream is not present") + var s stream + var ok bool + if isDelta { + s, ok = a.deltaStreams[streamID] + } else { + s, ok = a.streams[streamID] } - - if s.nodeID == "" { - s.nodeID = req.NodeId() - } - - if s.nodeID != req.NodeId() { - return stream{}, errors.Errorf("stream was authenticated for ID %s. Received request is for node with ID %s. Node ID cannot be changed after stream is initialized", s.nodeID, req.NodeId()) - } - - if s.resource == nil { - md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) - res, err := a.resource(user.Ctx(s.ctx, user.ControlPlane), md, req.NodeId()) - if err != nil { - return stream{}, err - } - s.resource = res - } - return s, nil -} - -func (a *authCallbacks) stream(streamID core_xds.StreamID, req util_xds.DiscoveryRequest) (stream, error) { - a.RLock() - s, ok := a.streams[streamID] a.RUnlock() if !ok { return stream{}, errors.New("stream is not present") diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index bcd1e353be9f..21338d2c33ac 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -51,5 +51,5 @@ type configParameters struct { IsGatewayDataplane bool Resources types.ProxyResources SystemCaPath string - UseDelta bool + UseDelta bool } diff --git a/pkg/xds/envoy/imports.go b/pkg/xds/envoy/imports.go index d3081bc99d0a..53b713f35492 100644 --- a/pkg/xds/envoy/imports.go +++ b/pkg/xds/envoy/imports.go @@ -127,6 +127,7 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/data/tap/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/filters/cel/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/fluentd/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/open_telemetry/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/stream/v3" @@ -151,12 +152,14 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/dependency/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/matcher/action/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/set_filter_state/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/adaptive_concurrency/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/admission_control/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/alternate_protocols_cache/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/aws_lambda/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/aws_request_signing/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/bandwidth_limit/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/basic_auth/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/buffer/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cache/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cdn_loop/v3" @@ -164,6 +167,7 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/compressor/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/connect_grpc_bridge/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/cors/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/credential_injector/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/csrf/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/custom_response/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/decompressor/v3" @@ -193,13 +197,16 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/oauth2/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/on_demand/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/original_src/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/proto_message_logging/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rate_limit_quota/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ratelimit/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_filter_state/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/set_metadata/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/stateful_session/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/tap/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/thrift_to_metadata/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/upstream_codec/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/wasm/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/http_inspector/v3" @@ -214,12 +221,19 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/dubbo_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/echo/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/ext_authz/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/action/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/codecs/dubbo/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/codecs/http1/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/matcher/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/router/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/generic_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/local_ratelimit/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/mongo_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/ratelimit/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rbac/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/redis_proxy/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/set_filter_state/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_cluster/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/sni_dynamic_forward_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" @@ -231,11 +245,14 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/wasm/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/zookeeper_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/dns_filter/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/session/dynamic_forward_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/session/http_capsule/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/cel/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/metadata/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/formatter/req_without_query/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/geoip_providers/common/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/geoip_providers/maxmind/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/health_check/event_sinks/file/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/health_checkers/redis/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/health_checkers/thrift/v3" @@ -246,6 +263,8 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/early_header_mutation/header_mutation/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_formatters/preserve_case/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/header_validators/envoy_default/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/injected_credentials/generic/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/injected_credentials/oauth2/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/original_ip_detection/custom_header/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/original_ip_detection/xff/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/http/stateful_session/cookie/v3" @@ -270,13 +289,17 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/common_inputs/ssl/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/consistent_hashing/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/ip/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/metadata/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/matching/input_matchers/runtime_fraction/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/network/dns_resolver/apple/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/network/dns_resolver/cares/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/network/dns_resolver/getaddrinfo/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/network/socket_interface/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/outlier_detection_monitors/common/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/outlier_detection_monitors/consecutive_errors/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/path/match/uri_template/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/path/rewrite/uri_template/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/connection_debug_visitor/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/connection_id_generator/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/crypto_stream/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/proof_source/v3" @@ -293,9 +316,13 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/extensions/retry/host/omit_host_metadata/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/retry/host/previous_hosts/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/retry/priority/previous_priorities/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/router/cluster_specifiers/lua/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/stat_sinks/graphite_statsd/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/stat_sinks/open_telemetry/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/stat_sinks/wasm/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/string_matcher/lua/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/resource_detectors/v3" + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/tracers/opentelemetry/samplers/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/alts/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/http_11_proxy/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/internal_upstream/v3" diff --git a/pkg/xds/server/callbacks/1 b/pkg/xds/server/callbacks/1 new file mode 100644 index 000000000000..78e3ab1e1c54 --- /dev/null +++ b/pkg/xds/server/callbacks/1 @@ -0,0 +1,31 @@ +c.mu.RLock() // read access to the map of all ADS streams + defer c.mu.RUnlock() + + state := c.streams[streamID] + + state.mu.Lock() // write access to the per Dataplane info + defer state.mu.Unlock() + + // update Dataplane status + subscription := state.subscription + subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) + subscription.Status.Total.ResponsesSent++ + subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ + + log := statusTrackerLog.WithValues( + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "streamID", streamID, + "type", shortEnvoyType(req.GetTypeUrl()), + "resourceVersion", resp.VersionInfo(), + "requestedResourceNames", req.GetResourceNames(), + "resourceCount", len(resp.GetResources()), + ) + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues( + "subscriptionID", subscription.Id, + "nonce", resp.GetNonce(), + ) + } + + log.V(1).Info("config sent") \ No newline at end of file diff --git a/pkg/xds/server/callbacks/2 b/pkg/xds/server/callbacks/2 new file mode 100644 index 000000000000..22fef9cabed0 --- /dev/null +++ b/pkg/xds/server/callbacks/2 @@ -0,0 +1,30 @@ +c.mu.RLock() // read access to the map of all ADS streams + defer c.mu.RUnlock() + + state := c.deltaStreams[streamID] + + state.mu.Lock() // write access to the per Dataplane info + defer state.mu.Unlock() + + // update Dataplane status + subscription := state.subscription + subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) + subscription.Status.Total.ResponsesSent++ + subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ + + log := statusTrackerLog.WithValues( + "proxyName", state.dataplaneId.Name, + "mesh", state.dataplaneId.Mesh, + "streamID", streamID, + "type", shortEnvoyType(req.GetTypeUrl()), + "requestedResourceNames", req.GetResourceNamesSubscribe(), + "resourceCount", len(resp.GetResources()), + ) + if statusTrackerLog.V(1).Enabled() { + log = log.WithValues( + "subscriptionID", subscription.Id, + "nonce", resp.GetNonce(), + ) + } + + log.V(1).Info("config sent") \ No newline at end of file diff --git a/pkg/xds/server/callbacks/dataplane_callbacks.go b/pkg/xds/server/callbacks/dataplane_callbacks.go index e9d1569c34a1..59e88778f46c 100644 --- a/pkg/xds/server/callbacks/dataplane_callbacks.go +++ b/pkg/xds/server/callbacks/dataplane_callbacks.go @@ -34,17 +34,18 @@ type xdsCallbacks struct { util_xds.NoopCallbacks sync.RWMutex - dpStreams map[core_xds.StreamID]dpStream - dpDeltaStreams map[core_xds.StreamID]dpStream + dpStreams map[core_xds.StreamID]dpStream + dpDeltaStreams map[core_xds.StreamID]dpStream + // we don't need separate map for stream because we use here resource key activeStreams map[core_model.ResourceKey]int } func DataplaneCallbacksToXdsCallbacks(callbacks DataplaneCallbacks) util_xds.MultiXDSCallbacks { return &xdsCallbacks{ - callbacks: callbacks, - dpStreams: map[core_xds.StreamID]dpStream{}, - dpDeltaStreams: map[core_xds.StreamID]dpStream{}, - activeStreams: map[core_model.ResourceKey]int{}, + callbacks: callbacks, + dpStreams: map[core_xds.StreamID]dpStream{}, + dpDeltaStreams: map[core_xds.StreamID]dpStream{}, + activeStreams: map[core_model.ResourceKey]int{}, } } @@ -55,97 +56,39 @@ type dpStream struct { var _ util_xds.MultiXDSCallbacks = &xdsCallbacks{} - func (d *xdsCallbacks) OnStreamClosed(streamID core_xds.StreamID) { - var lastStreamDpKey *core_model.ResourceKey - d.Lock() - dpStream := d.dpStreams[streamID] - if dpKey := dpStream.dp; dpKey != nil { - d.activeStreams[*dpKey]-- - if d.activeStreams[*dpKey] == 0 { - lastStreamDpKey = dpKey - delete(d.activeStreams, *dpKey) - } - } - delete(d.dpStreams, streamID) - d.Unlock() - if lastStreamDpKey != nil { - // execute callback after lock is freed, so heavy callback implementation won't block every callback for every DPP. - d.callbacks.OnProxyDisconnected(dpStream.ctx, streamID, *lastStreamDpKey) - } + d.onStreamClosed(streamID, false) } -func (d *xdsCallbacks) OnStreamRequest(streamID core_xds.StreamID, request util_xds.DiscoveryRequest) error { - if request.NodeId() == "" { - // from https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-versioning: - // Only the first request on a stream is guaranteed to carry the node identifier. - // The subsequent discovery requests on the same stream may carry an empty node identifier. - // This holds true regardless of the acceptance of the discovery responses on the same stream. - // The node identifier should always be identical if present more than once on the stream. - // It is sufficient to only check the first message for the node identifier as a result. - return nil - } - - d.RLock() - alreadyProcessed := d.dpStreams[streamID].dp != nil - d.RUnlock() - if alreadyProcessed { - return nil - } - - proxyId, err := core_xds.ParseProxyIdFromString(request.NodeId()) - if err != nil { - return errors.Wrap(err, "invalid node ID") - } - dpKey := proxyId.ToResourceKey() - metadata := core_xds.DataplaneMetadataFromXdsMetadata(request.Metadata()) - if metadata == nil { - return errors.New("metadata in xDS Node cannot be nil") - } - - d.Lock() - // in case client will open 2 concurrent request for the same streamID then - // we don't to increment the counter twice, so checking once again that stream - // wasn't processed - alreadyProcessed = d.dpStreams[streamID].dp != nil - if alreadyProcessed { - return nil - } +func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { + d.onStreamClosed(streamID, true) +} - dpStream := d.dpStreams[streamID] - dpStream.dp = &dpKey - d.dpStreams[streamID] = dpStream +func (d *xdsCallbacks) OnStreamRequest(streamID core_xds.StreamID, request util_xds.DiscoveryRequest) error { + return d.onStreamRequest(streamID, request, false) +} - activeStreams := d.activeStreams[dpKey] - d.activeStreams[dpKey]++ - d.Unlock() +func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request util_xds.DeltaDiscoveryRequest) error { + return d.onStreamRequest(streamID, request, true) +} - if activeStreams == 0 { - if err := d.callbacks.OnProxyConnected(streamID, dpKey, dpStream.ctx, *metadata); err != nil { - return err - } - } else { - if err := d.callbacks.OnProxyReconnected(streamID, dpKey, dpStream.ctx, *metadata); err != nil { - return err - } - } - return nil +func (d *xdsCallbacks) OnStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { + return d.onStreamOpen(ctx, streamID, false) } func (d *xdsCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { - d.Lock() - defer d.Unlock() - dps := dpStream{ - ctx: ctx, - } - d.dpDeltaStreams[streamID] = dps - return nil + return d.onStreamOpen(ctx, streamID, true) } -func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { +func (d *xdsCallbacks) onStreamClosed(streamID core_xds.StreamID, isDelta bool) { var lastStreamDpKey *core_model.ResourceKey d.Lock() - dpStream := d.dpDeltaStreams[streamID] + var dpStream dpStream + if isDelta { + dpStream = d.dpDeltaStreams[streamID] + } else { + dpStream = d.dpStreams[streamID] + } if dpKey := dpStream.dp; dpKey != nil { d.activeStreams[*dpKey]-- if d.activeStreams[*dpKey] == 0 { @@ -153,7 +96,11 @@ func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { delete(d.activeStreams, *dpKey) } } - delete(d.dpDeltaStreams, streamID) + if isDelta { + delete(d.dpDeltaStreams, streamID) + } else { + delete(d.dpStreams, streamID) + } d.Unlock() if lastStreamDpKey != nil { // execute callback after lock is freed, so heavy callback implementation won't block every callback for every DPP. @@ -161,7 +108,7 @@ func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { } } -func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request util_xds.DeltaDiscoveryRequest) error { +func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_xds.Request, isDelta bool) error { if request.NodeId() == "" { // from https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-versioning: // Only the first request on a stream is guaranteed to carry the node identifier. @@ -173,7 +120,13 @@ func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request } d.RLock() - alreadyProcessed := d.dpDeltaStreams[streamID].dp != nil + var alreadyProcessed bool + if isDelta { + alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil + } else { + alreadyProcessed = d.dpStreams[streamID].dp != nil + } + d.RUnlock() if alreadyProcessed { return nil @@ -193,14 +146,27 @@ func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request // in case client will open 2 concurrent request for the same streamID then // we don't to increment the counter twice, so checking once again that stream // wasn't processed - alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil + if isDelta { + alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil + } else { + alreadyProcessed = d.dpStreams[streamID].dp != nil + } if alreadyProcessed { return nil } - dpStream := d.dpDeltaStreams[streamID] + var dpStream dpStream + if isDelta { + dpStream = d.dpDeltaStreams[streamID] + } else { + dpStream = d.dpStreams[streamID] + } dpStream.dp = &dpKey - d.dpDeltaStreams[streamID] = dpStream + if isDelta { + d.dpDeltaStreams[streamID] = dpStream + } else { + d.dpStreams[streamID] = dpStream + } activeStreams := d.activeStreams[dpKey] d.activeStreams[dpKey]++ @@ -218,13 +184,18 @@ func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request return nil } -func (d *xdsCallbacks) OnStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { +func (d *xdsCallbacks) onStreamOpen(ctx context.Context, streamID core_xds.StreamID, isDelta bool) error { d.Lock() defer d.Unlock() dps := dpStream{ ctx: ctx, } - d.dpStreams[streamID] = dps + if isDelta { + d.dpDeltaStreams[streamID] = dps + } else { + d.dpStreams[streamID] = dps + } + return nil } diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index 2e606cd2b284..bc23961c9514 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -39,7 +39,7 @@ func NewDataplaneStatusTracker( runtimeInfo: runtimeInfo, createStatusSink: createStatusSink, streams: make(map[int64]*streamState), - deltaStreams: make(map[int64]*streamState), + deltaStreams: make(map[int64]*streamState), } } @@ -51,7 +51,7 @@ type dataplaneStatusTracker struct { createStatusSink DataplaneInsightSinkFactoryFunc mu sync.RWMutex // protects access to the fields below streams map[int64]*streamState - deltaStreams map[int64]*streamState + deltaStreams map[int64]*streamState } type streamState struct { @@ -64,73 +64,83 @@ type streamState struct { // OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS). // Returning an error will end processing and close the stream. OnStreamClosed will still be called. func (c *dataplaneStatusTracker) OnStreamOpen(ctx context.Context, streamID int64, typ string) error { - c.mu.Lock() // write access to the map of all ADS streams - defer c.mu.Unlock() - - // initialize subscription - now := core.Now() - subscription := &mesh_proto.DiscoverySubscription{ - Id: core.NewUUID(), - ControlPlaneInstanceId: c.runtimeInfo.GetInstanceId(), - ConnectTime: util_proto.MustTimestampProto(now), - Status: mesh_proto.NewSubscriptionStatus(now), - Version: mesh_proto.NewVersion(), - } - // initialize state per ADS stream - state := &streamState{ - stop: make(chan struct{}), - subscription: subscription, - } - // save - c.streams[streamID] = state + return c.onStreamOpen(streamID, typ, false) +} - statusTrackerLog.V(1).Info("proxy connecting", "streamID", streamID, "type", typ, "subscriptionID", subscription.Id) - return nil +// OnDeltaStreamOpen is called once an Delta xDS stream is open with a stream ID and the type URL (or "" for ADS). +// Returning an error will end processing and close the stream. OnDeltaStreamOpen will still be called. +func (c *dataplaneStatusTracker) OnDeltaStreamOpen(_ context.Context, streamID int64, typ string) error { + return c.onStreamOpen(streamID, typ, true) } // OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID. func (c *dataplaneStatusTracker) OnStreamClosed(streamID int64) { - c.mu.Lock() // write access to the map of all ADS streams - defer c.mu.Unlock() + c.onStreamClose(streamID, false) +} - state := c.streams[streamID] - if state == nil { - statusTrackerLog.Info("[WARNING] proxy disconnected but no state in the status_tracker", "streamID", streamID) - return - } +// OnDeltaStreamClosed is called immediately prior to closing an Delta xDS stream with a stream ID. +func (c *dataplaneStatusTracker) OnDeltaStreamClosed(streamID int64) { + c.onStreamClose(streamID, true) +} - delete(c.streams, streamID) +// OnStreamRequest is called once a request is received on a stream. +// Returning an error will end processing and close the stream. OnStreamClosed will still be called. +func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.DiscoveryRequest) error { + return c.onStreamRequest(streamID, req, false) +} - // finilize subscription - state.mu.Lock() // write access to the per Dataplane info - subscription := state.subscription - subscription.DisconnectTime = util_proto.MustTimestampProto(core.Now()) - state.mu.Unlock() +// OnStreamDeltaRequest is called once a request is received on a delta stream. +// Returning an error will end processing and close the stream. OnStreamDeltaRequest will still be called. +func (c *dataplaneStatusTracker) OnStreamDeltaRequest(streamID int64, req util_xds.DeltaDiscoveryRequest) error { + return c.onStreamRequest(streamID, req, true) +} - // trigger final flush - state.Close() +// OnStreamResponse is called immediately prior to sending a response on a stream. +func (c *dataplaneStatusTracker) OnStreamResponse(streamID int64, req util_xds.DiscoveryRequest, resp util_xds.DiscoveryResponse) { + c.onStreamResponse(streamID, req, resp, false) +} - log := statusTrackerLog.WithValues( - "streamID", streamID, - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "subscriptionID", state.subscription.Id, - ) +// OnStreamDeltaResponse is called immediately prior to sending a response on a delta stream. +func (c *dataplaneStatusTracker) OnStreamDeltaResponse(streamID int64, req util_xds.DeltaDiscoveryRequest, resp util_xds.DeltaDiscoveryResponse) { + c.onStreamResponse(streamID, req, resp, true) +} - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues("subscription", subscription) +// To keep logs short, we want to log "Listeners" instead of full qualified Envoy type url name +func shortEnvoyType(typeURL string) string { + segments := strings.Split(typeURL, ".") + if len(segments) <= 1 { + return typeURL } + return segments[len(segments)-1] +} - log.Info("proxy disconnected") +func (c *dataplaneStatusTracker) GetStatusAccessor(streamID int64) (SubscriptionStatusAccessor, bool) { + state, ok := c.streams[streamID] + return state, ok } -// OnStreamRequest is called once a request is received on a stream. -// Returning an error will end processing and close the stream. OnStreamClosed will still be called. -func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.DiscoveryRequest) error { +var _ SubscriptionStatusAccessor = &streamState{} + +func (s *streamState) GetStatus() (core_model.ResourceKey, *mesh_proto.DiscoverySubscription) { + s.mu.RLock() // read access to the per Dataplane info + defer s.mu.RUnlock() + return s.dataplaneId, proto.Clone(s.subscription).(*mesh_proto.DiscoverySubscription) +} + +func (s *streamState) Close() { + close(s.stop) +} + +func (c *dataplaneStatusTracker) onStreamRequest(streamID int64, req util_xds.Request, isDelta bool) error { c.mu.RLock() // read access to the map of all ADS streams defer c.mu.RUnlock() - state := c.streams[streamID] + var state *streamState + if isDelta { + state = c.deltaStreams[streamID] + } else { + state = c.streams[streamID] + } state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() @@ -218,12 +228,16 @@ func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.Di return nil } -// OnStreamResponse is called immediately prior to sending a response on a stream. -func (c *dataplaneStatusTracker) OnStreamResponse(streamID int64, req util_xds.DiscoveryRequest, resp util_xds.DiscoveryResponse) { +func (c *dataplaneStatusTracker) onStreamResponse(streamID int64, req util_xds.Request, resp util_xds.Response, isDelta bool) { c.mu.RLock() // read access to the map of all ADS streams defer c.mu.RUnlock() - state := c.streams[streamID] + var state *streamState + if isDelta { + state = c.deltaStreams[streamID] + } else { + state = c.streams[streamID] + } state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() @@ -241,7 +255,7 @@ func (c *dataplaneStatusTracker) OnStreamResponse(streamID int64, req util_xds.D "type", shortEnvoyType(req.GetTypeUrl()), "resourceVersion", resp.VersionInfo(), "requestedResourceNames", req.GetResourceNames(), - "resourceCount", len(resp.GetResources()), + "resourceCount", resp.GetNumberOfResources(), ) if statusTrackerLog.V(1).Enabled() { log = log.WithValues( @@ -253,35 +267,7 @@ func (c *dataplaneStatusTracker) OnStreamResponse(streamID int64, req util_xds.D log.V(1).Info("config sent") } -// To keep logs short, we want to log "Listeners" instead of full qualified Envoy type url name -func shortEnvoyType(typeURL string) string { - segments := strings.Split(typeURL, ".") - if len(segments) <= 1 { - return typeURL - } - return segments[len(segments)-1] -} - -func (c *dataplaneStatusTracker) GetStatusAccessor(streamID int64) (SubscriptionStatusAccessor, bool) { - state, ok := c.streams[streamID] - return state, ok -} - -var _ SubscriptionStatusAccessor = &streamState{} - -func (s *streamState) GetStatus() (core_model.ResourceKey, *mesh_proto.DiscoverySubscription) { - s.mu.RLock() // read access to the per Dataplane info - defer s.mu.RUnlock() - return s.dataplaneId, proto.Clone(s.subscription).(*mesh_proto.DiscoverySubscription) -} - -func (s *streamState) Close() { - close(s.stop) -} - -// OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS). -// Returning an error will end processing and close the stream. OnStreamClosed will still be called. -func (c *dataplaneStatusTracker) OnDeltaStreamOpen(ctx context.Context, streamID int64, typ string) error { +func (c *dataplaneStatusTracker) onStreamOpen(streamID int64, typ string, isDelta bool) error { c.mu.Lock() // write access to the map of all ADS streams defer c.mu.Unlock() @@ -300,25 +286,36 @@ func (c *dataplaneStatusTracker) OnDeltaStreamOpen(ctx context.Context, streamID subscription: subscription, } // save - c.deltaStreams[streamID] = state + if isDelta { + c.deltaStreams[streamID] = state + } else { + c.streams[streamID] = state + } statusTrackerLog.V(1).Info("proxy connecting", "streamID", streamID, "type", typ, "subscriptionID", subscription.Id) return nil } -// OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID. -func (c *dataplaneStatusTracker) OnDeltaStreamClosed(streamID int64) { +func (c *dataplaneStatusTracker) onStreamClose(streamID int64, isDelta bool) { c.mu.Lock() // write access to the map of all ADS streams defer c.mu.Unlock() - state := c.deltaStreams[streamID] + var state *streamState + if isDelta { + state = c.deltaStreams[streamID] + } else { + state = c.streams[streamID] + } if state == nil { statusTrackerLog.Info("[WARNING] proxy disconnected but no state in the status_tracker", "streamID", streamID) return } - delete(c.deltaStreams, streamID) - + if isDelta { + delete(c.deltaStreams, streamID) + } else { + delete(c.streams, streamID) + } // finilize subscription state.mu.Lock() // write access to the per Dataplane info subscription := state.subscription @@ -341,131 +338,3 @@ func (c *dataplaneStatusTracker) OnDeltaStreamClosed(streamID int64) { log.Info("proxy disconnected") } - -// OnStreamRequest is called once a request is received on a stream. -// Returning an error will end processing and close the stream. OnStreamClosed will still be called. -func (c *dataplaneStatusTracker) OnStreamDeltaRequest(streamID int64, req util_xds.DeltaDiscoveryRequest) error { - c.mu.RLock() // read access to the map of all ADS streams - defer c.mu.RUnlock() - - state := c.deltaStreams[streamID] - - state.mu.Lock() // write access to the per Dataplane info - defer state.mu.Unlock() - - if state.dataplaneId == (core_model.ResourceKey{}) { - // Infer the Dataplane ID. - if proxyId, err := core_xds.ParseProxyIdFromString(req.NodeId()); err == nil { - state.dataplaneId = proxyId.ToResourceKey() - var dpType core_model.ResourceType - md := core_xds.DataplaneMetadataFromXdsMetadata(req.Metadata()) - - // If the dataplane was started with a resource YAML, then it - // will be serialized in the node metadata and we would know - // the underlying type directly. Since that is optional, we - // can't depend on it here, so we map from the proxy type, - // which is guaranteed. - switch md.GetProxyType() { - case mesh_proto.IngressProxyType: - dpType = core_mesh.ZoneIngressType - case mesh_proto.DataplaneProxyType: - dpType = core_mesh.DataplaneType - case mesh_proto.EgressProxyType: - dpType = core_mesh.ZoneEgressType - } - - log := statusTrackerLog.WithValues( - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "streamID", streamID, - "type", md.GetProxyType(), - "dpVersion", md.GetVersion().GetKumaDp().GetVersion(), - "subscriptionID", state.subscription.Id, - ) - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues("node", req.Node()) - } - log.Info("proxy connected") - - if md.GetVersion() != nil { - state.subscription.Version = md.GetVersion() - } else { - statusTrackerLog.Error(err, "failed to extract version out of the Envoy metadata", "streamid", streamID, "metadata", req.Metadata()) - } - // Kick off the async Dataplane status flusher. - go c.createStatusSink(dpType, state).Start(state.stop) - } else { - statusTrackerLog.Error(err, "failed to parse Dataplane Id out of DiscoveryRequest", "streamid", streamID, "req", req) - } - } - - subscription := state.subscription - log := statusTrackerLog.WithValues( - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "streamID", streamID, - "type", shortEnvoyType(req.GetTypeUrl()), - "resourceVersion", req.GetInitialResourceVersions(), - ) - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues( - "resourceNames", req.GetResourceNamesSubscribe(), - "subscriptionID", subscription.Id, - "nonce", req.GetResponseNonce(), - ) - } - - // update Dataplane status - if req.GetResponseNonce() != "" { - subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) - if req.HasErrors() { - log.Info("config rejected") - subscription.Status.Total.ResponsesRejected++ - subscription.Status.StatsOf(req.GetTypeUrl()).ResponsesRejected++ - } else { - log.V(1).Info("config accepted") - subscription.Status.Total.ResponsesAcknowledged++ - subscription.Status.StatsOf(req.GetTypeUrl()).ResponsesAcknowledged++ - } - } else { - if !statusTrackerLog.V(1).Enabled() { // it was already added, no need to add it twice - log = log.WithValues("resourceNames", req.GetResourceNamesSubscribe()) - } - log.Info("config requested") - } - return nil -} - -// OnStreamResponse is called immediately prior to sending a response on a stream. -func (c *dataplaneStatusTracker) OnStreamDeltaResponse(streamID int64, req util_xds.DeltaDiscoveryRequest, resp util_xds.DeltaDiscoveryResponse) { - c.mu.RLock() // read access to the map of all ADS streams - defer c.mu.RUnlock() - - state := c.deltaStreams[streamID] - - state.mu.Lock() // write access to the per Dataplane info - defer state.mu.Unlock() - - // update Dataplane status - subscription := state.subscription - subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) - subscription.Status.Total.ResponsesSent++ - subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ - - log := statusTrackerLog.WithValues( - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "streamID", streamID, - "type", shortEnvoyType(req.GetTypeUrl()), - "requestedResourceNames", req.GetResourceNamesSubscribe(), - "resourceCount", len(resp.GetResources()), - ) - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues( - "subscriptionID", subscription.Id, - "nonce", resp.GetNonce(), - ) - } - - log.V(1).Info("config sent") -} diff --git a/pkg/xds/server/callbacks/nack_backoff.go b/pkg/xds/server/callbacks/nack_backoff.go index fe98ef62ced6..76dcdfd84bfb 100644 --- a/pkg/xds/server/callbacks/nack_backoff.go +++ b/pkg/xds/server/callbacks/nack_backoff.go @@ -23,23 +23,14 @@ func NewNackBackoff(backoff time.Duration) util_xds.MultiXDSCallbacks { } func (n *nackBackoff) OnStreamResponse(_ int64, request util_xds.DiscoveryRequest, _ util_xds.DiscoveryResponse) { - if request.HasErrors() { - // When DiscoveryRequest contains errors, it means that Envoy rejected configuration generated by Control Plane - // It may happen for several reasons: - // 1) Eventual consistency - ex. listener consists reference to cluster which does not exist because listener was send before cluster (there is no ordering of responses) - // 2) Config is valid from CP side but invalid from Envoy side - ex. something already listening at this address:port - // - // Second case is especially dangerous because we will end up in a loop. - // CP is constantly trying to send a config and Envoy immediately rejects the config. - // Without this backoff, CP is under a lot of pressure from faulty Envoy. - // - // It is safe to sleep here because OnStreamResponse is executed in the goroutine of a single ADS stream - nackLog.Info("config was previously rejected by Envoy. Applying backoff before resending it", "backoff", n.backoff, "nodeID", request.NodeId(), "reason", request.ErrorMsg()) - time.Sleep(n.backoff) - } + n.applyBackoff(request) } func (n *nackBackoff) OnStreamDeltaResponse(_ int64, request util_xds.DeltaDiscoveryRequest, _ util_xds.DeltaDiscoveryResponse) { + n.applyBackoff(request) +} + +func (n *nackBackoff) applyBackoff(request util_xds.Request) { if request.HasErrors() { // When DiscoveryRequest contains errors, it means that Envoy rejected configuration generated by Control Plane // It may happen for several reasons: @@ -55,4 +46,3 @@ func (n *nackBackoff) OnStreamDeltaResponse(_ int64, request util_xds.DeltaDisco time.Sleep(n.backoff) } } - diff --git a/pkg/xds/server/v3/components.go b/pkg/xds/server/v3/components.go index 7ec520980bf7..c8767b6213fa 100644 --- a/pkg/xds/server/v3/components.go +++ b/pkg/xds/server/v3/components.go @@ -6,11 +6,10 @@ import ( envoy_service_discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" "github.com/envoyproxy/go-control-plane/pkg/server/config" - envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v3" - envoy_server_delta "github.com/envoyproxy/go-control-plane/pkg/server/delta/v3" envoy_server_rest "github.com/envoyproxy/go-control-plane/pkg/server/rest/v3" envoy_server_sotw "github.com/envoyproxy/go-control-plane/pkg/server/sotw/v3" + envoy_server "github.com/envoyproxy/go-control-plane/pkg/server/v3" mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/core" @@ -81,14 +80,12 @@ func RegisterXDS( ), util_xds_v3.AdaptDeltaCallbacks(DefaultDataplaneStatusTracker(rt, envoyCpCtx.Secrets)), util_xds_v3.AdaptDeltaCallbacks(xds_callbacks.NewNackBackoff(rt.Config().XdsServer.NACKBackoff.Duration)), - newResourceWarmingForcer(xdsContext.Cache(), xdsContext.Hasher()), } if cb := rt.XDS().ServerCallbacks; cb != nil { deltaCallbacks = append(deltaCallbacks, util_xds_v3.AdaptDeltaCallbacks(cb)) } - rest := envoy_server_rest.NewServer(xdsContext.Cache(), callbacks) sotw := envoy_server_sotw.NewServer(context.Background(), xdsContext.Cache(), callbacks) ordered := func(o *config.Opts) { diff --git a/pkg/xds/server/v3/resource_warming_forcer.go b/pkg/xds/server/v3/resource_warming_forcer.go index ca5821baeced..5064420faf53 100644 --- a/pkg/xds/server/v3/resource_warming_forcer.go +++ b/pkg/xds/server/v3/resource_warming_forcer.go @@ -84,13 +84,6 @@ func (r *resourceWarmingForcer) OnStreamClosed(streamID int64, _ *envoy_core.Nod delete(r.nodeIDs, streamID) } -func (r *resourceWarmingForcer) OnDeltaStreamClosed(streamID int64, _ *envoy_core.Node) { - r.Lock() - defer r.Unlock() - delete(r.lastEndpointNonces, streamID) - delete(r.nodeIDs, streamID) -} - func (r *resourceWarmingForcer) OnStreamRequest(streamID xds.StreamID, request *envoy_sd.DiscoveryRequest) error { if request.TypeUrl != envoy_resource.EndpointType { return nil // we force Cluster warming only on receiving the same EDS Discovery Request @@ -121,57 +114,6 @@ func (r *resourceWarmingForcer) OnStreamRequest(streamID xds.StreamID, request * return nil } -func (r *resourceWarmingForcer) OnStreamDeltaRequest(streamID xds.StreamID, request *envoy_sd.DeltaDiscoveryRequest) error { - warmingForcerLog.Info("check something", "streamdi", streamID, "request.TypeUrl", request.TypeUrl, "request.ResponseNonce", request.ResponseNonce) - if request.TypeUrl != envoy_resource.EndpointType { - return nil // we force Cluster warming only on receiving the same EDS Discovery Request - } - if request.ResponseNonce == "" { - return nil // initial request, no need to force warming - } - if request.ErrorDetail != nil { - return nil // we only care about ACKs, otherwise we can get 2 Nonces with multiple NACKs - } - warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") - r.Lock() - lastEndpointNonce := r.lastEndpointNonces[streamID] - r.lastEndpointNonces[streamID] = request.ResponseNonce - nodeID := r.nodeIDs[streamID] - if nodeID == "" { - nodeID = r.hasher.ID(request.Node) // request.Node can be set only on first request therefore we need to save it - r.nodeIDs[streamID] = nodeID - } - r.Unlock() - warmingForcerLog.Info("HMMM", "lastEndpointNonce", lastEndpointNonce, "request.ResponseNonce ", request.ResponseNonce ) - if lastEndpointNonce == request.ResponseNonce || lastEndpointNonce == "" { - warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") - if err := r.deltaForceNewEndpointsVersion(nodeID); err != nil { - warmingForcerLog.Error(err, "could not force cluster warming") - } - } - return nil -} - -func (r *resourceWarmingForcer) OnStreamDeltaResponse(streamID int64, request *envoy_sd.DeltaDiscoveryRequest, _ *envoy_sd.DeltaDiscoveryResponse) { - if request.TypeUrl != envoy_resource.ClusterType { - return - } - r.Lock() - - nodeID := r.nodeIDs[streamID] - if nodeID == "" { - nodeID = r.hasher.ID(request.Node) // request.Node can be set only on first request therefore we need to save it - r.nodeIDs[streamID] = nodeID - } - r.Unlock() - if request.TypeUrl == envoy_resource.ClusterType { - warmingForcerLog.Info("received second Endpoint DiscoveryRequest with same Nonce. Forcing new version of Endpoints to warm the Cluster") - if err := r.deltaForceNewEndpointsVersion(nodeID); err != nil { - warmingForcerLog.Error(err, "could not force cluster warming") - } - } -} - func (r *resourceWarmingForcer) forceNewEndpointsVersion(nodeID string) error { snapshot, err := r.cache.GetSnapshot(nodeID) if err != nil { @@ -190,24 +132,4 @@ func (r *resourceWarmingForcer) forceNewEndpointsVersion(nodeID string) error { return nil } -func (r *resourceWarmingForcer) deltaForceNewEndpointsVersion(nodeID string) error { - snapshot, err := r.cache.GetSnapshot(nodeID) - if err != nil { - return nil // GetSnapshot returns an error if there is no snapshot. We don't need to force on a new snapshot - } - cacheSnapshot, ok := snapshot.(*envoy_cache.Snapshot) - if !ok { - return errors.New("couldn't convert snapshot from cache to envoy Snapshot") - } - responseType, err := envoy_cache.GetResponseTypeURL(types.Endpoint) - if err != nil { - return err - } - for _, endpoint := range cacheSnapshot.VersionMap[responseType]{ - cacheSnapshot.VersionMap[responseType][endpoint] = "" - } - - return nil -} - var _ envoy_xds.Callbacks = &resourceWarmingForcer{} diff --git a/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go b/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go index 80aa977fb03a..610c8e326333 100644 --- a/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go +++ b/test/e2e_env/kubernetes/meshcircuitbreaker/meshcircuitbreaker.go @@ -2,7 +2,6 @@ package meshcircuitbreaker import ( "fmt" - "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -120,7 +119,6 @@ spec: // when Expect(kubernetes.Cluster.Install(YamlK8s(config))).To(Succeed()) - time.Sleep(1*time.Hour) // then Eventually(func(g Gomega) ([]client.FailureResponse, error) { return client.CollectResponsesAndFailures( @@ -138,7 +136,7 @@ spec: ContainElement(HaveField("ResponseCode", 503)), )) }, - FEntry("outbound circuit breaker", fmt.Sprintf(` + Entry("outbound circuit breaker", fmt.Sprintf(` apiVersion: kuma.io/v1alpha1 kind: MeshCircuitBreaker metadata: From af6ae16be4ea6eb40d66a46de1292fbdc0f6c439 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 3 Sep 2024 08:48:27 -0500 Subject: [PATCH 03/12] cleanup Signed-off-by: Lukasz Dziedziak --- pkg/xds/server/callbacks/1 | 31 ------------------------------- pkg/xds/server/callbacks/2 | 30 ------------------------------ 2 files changed, 61 deletions(-) delete mode 100644 pkg/xds/server/callbacks/1 delete mode 100644 pkg/xds/server/callbacks/2 diff --git a/pkg/xds/server/callbacks/1 b/pkg/xds/server/callbacks/1 deleted file mode 100644 index 78e3ab1e1c54..000000000000 --- a/pkg/xds/server/callbacks/1 +++ /dev/null @@ -1,31 +0,0 @@ -c.mu.RLock() // read access to the map of all ADS streams - defer c.mu.RUnlock() - - state := c.streams[streamID] - - state.mu.Lock() // write access to the per Dataplane info - defer state.mu.Unlock() - - // update Dataplane status - subscription := state.subscription - subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) - subscription.Status.Total.ResponsesSent++ - subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ - - log := statusTrackerLog.WithValues( - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "streamID", streamID, - "type", shortEnvoyType(req.GetTypeUrl()), - "resourceVersion", resp.VersionInfo(), - "requestedResourceNames", req.GetResourceNames(), - "resourceCount", len(resp.GetResources()), - ) - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues( - "subscriptionID", subscription.Id, - "nonce", resp.GetNonce(), - ) - } - - log.V(1).Info("config sent") \ No newline at end of file diff --git a/pkg/xds/server/callbacks/2 b/pkg/xds/server/callbacks/2 deleted file mode 100644 index 22fef9cabed0..000000000000 --- a/pkg/xds/server/callbacks/2 +++ /dev/null @@ -1,30 +0,0 @@ -c.mu.RLock() // read access to the map of all ADS streams - defer c.mu.RUnlock() - - state := c.deltaStreams[streamID] - - state.mu.Lock() // write access to the per Dataplane info - defer state.mu.Unlock() - - // update Dataplane status - subscription := state.subscription - subscription.Status.LastUpdateTime = util_proto.MustTimestampProto(core.Now()) - subscription.Status.Total.ResponsesSent++ - subscription.Status.StatsOf(resp.GetTypeUrl()).ResponsesSent++ - - log := statusTrackerLog.WithValues( - "proxyName", state.dataplaneId.Name, - "mesh", state.dataplaneId.Mesh, - "streamID", streamID, - "type", shortEnvoyType(req.GetTypeUrl()), - "requestedResourceNames", req.GetResourceNamesSubscribe(), - "resourceCount", len(resp.GetResources()), - ) - if statusTrackerLog.V(1).Enabled() { - log = log.WithValues( - "subscriptionID", subscription.Id, - "nonce", resp.GetNonce(), - ) - } - - log.V(1).Info("config sent") \ No newline at end of file From f23136423609e14f55c30b3f0d35330682422b6a Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 3 Sep 2024 09:20:13 -0500 Subject: [PATCH 04/12] more cleanup and tests Signed-off-by: Lukasz Dziedziak --- .../testdata/bootstrap-request-0.golden.json | 3 +- .../testdata/bootstrap-request-1.golden.json | 3 +- .../testdata/bootstrap-request-2.golden.json | 3 +- .../testdata/bootstrap-request-3.golden.json | 3 +- pkg/config/app/kuma-cp/config.go | 2 + pkg/config/app/kuma-dp/config.go | 9 +- .../testdata/invalid-config.golden.txt | 2 +- .../testdata/invalid-config.input.yaml | 1 + .../kuma-dp/testdata/valid-config.input.yaml | 1 + pkg/config/loader_test.go | 3 + pkg/xds/bootstrap/components.go | 1 + pkg/xds/bootstrap/generator.go | 10 +- pkg/xds/bootstrap/generator_test.go | 36 ++++- pkg/xds/bootstrap/server_test.go | 2 +- pkg/xds/bootstrap/template_v3.go | 10 +- ...nfig-minimal-request-and-delta.golden.yaml | 145 ++++++++++++++++++ 16 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json index 6f63ce578d65..faa20220fbcb 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json @@ -31,5 +31,6 @@ "certPath": "/tmp/cert.pem", "keyPath": "/tmp/key.pem" }, - "systemCaPath": "" + "systemCaPath": "", + "xdsConfigType": "sotw" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json index 5e5d5da29c45..f5da93dc81d9 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json @@ -30,5 +30,6 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "" + "systemCaPath": "", + "xdsConfigType": "sotw" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json index 473d5360187d..ec718bd63d64 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json @@ -29,5 +29,6 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "" + "systemCaPath": "", + "xdsConfigType": "sotw" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json index 473d5360187d..ec718bd63d64 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json @@ -29,5 +29,6 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "" + "systemCaPath": "", + "xdsConfigType": "sotw" } diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index 5350c82c2efd..0768f4f67cba 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -464,6 +464,8 @@ type ExperimentalConfig struct { // If true skips persisted VIPs. Change to true only if generateMeshServices is enabled. // Do not enable on production. SkipPersistedVIPs bool `json:"skipPersistedVIPs" envconfig:"KUMA_EXPERIMENTAL_SKIP_PERSISTED_VIPS"` + // If true uses Delta xDS to deliver changes to sidecars. + UseDeltaXDS bool `json:"useDeltaXDS" envconfig:"KUMA_EXPERIMENTAL_USE_DELTA_XDS"` } type ExperimentalKDSEventBasedWatchdog struct { diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index cffa8e37f8e7..a30da65b0eba 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -217,7 +217,7 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` - // XDSConfigType + // XDSConfigType defines xDS communication type between Envoy and control-plane XDSConfigType string `json:"xdsConfigType,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_type"` } @@ -337,6 +337,13 @@ func (d *DataplaneRuntime) Validate() error { if d.BinaryPath == "" { errs = multierr.Append(errs, errors.Errorf(".BinaryPath must be non-empty")) } + if d.XDSConfigType != "" { + switch d.XDSConfigType { + case "delta", "sotw": + default: + errs = multierr.Append(errs, errors.Errorf(".XDSConfigType can be one of: delta, sotw")) + } + } return errs } diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt index 5ca12650bb24..59e5a9b2bf4a 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt @@ -1 +1 @@ -parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty +parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty; .XDSConfigType can be one of: delta, sotw diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml index 7467f0431afa..f2b1e736c7fb 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml @@ -12,3 +12,4 @@ dataplane: proxyType: not-a-proxy dataplaneRuntime: binaryPath: + xdsConfigType: a diff --git a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml index 5046ac0d7784..0ab02544046e 100644 --- a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml @@ -12,3 +12,4 @@ dataplaneRuntime: binaryPath: envoy.sh configDir: /var/run/envoy envoyLogLevel: trace + xdsConfigType: delta diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 9416bc566280..2d37a3be9f1e 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -370,6 +370,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Experimental.SidecarContainers).To(BeTrue()) Expect(cfg.Experimental.SkipPersistedVIPs).To(BeTrue()) Expect(cfg.Experimental.GenerateMeshServices).To(BeTrue()) + Expect(cfg.Experimental.UseDeltaXDS).To(BeTrue()) Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1)) Expect(cfg.EventBus.BufferSize).To(Equal(uint(30))) @@ -754,6 +755,7 @@ experimental: sidecarContainers: true generateMeshServices: true skipPersistedVIPs: true + useDeltaXDS: true proxy: gateway: globalDownstreamMaxConnections: 1 @@ -1058,6 +1060,7 @@ meshService: "KUMA_EXPERIMENTAL_SIDECAR_CONTAINERS": "true", "KUMA_EXPERIMENTAL_GENERATE_MESH_SERVICES": "true", "KUMA_EXPERIMENTAL_SKIP_PERSISTED_VIPS": "true", + "KUMA_EXPERIMENTAL_USE_DELTA_XDS": "true", "KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS": "1", "KUMA_TRACING_OPENTELEMETRY_ENDPOINT": "otel-collector:4317", "KUMA_TRACING_OPENTELEMETRY_ENABLED": "true", diff --git a/pkg/xds/bootstrap/components.go b/pkg/xds/bootstrap/components.go index 64dcc49197f0..afde7225440b 100644 --- a/pkg/xds/bootstrap/components.go +++ b/pkg/xds/bootstrap/components.go @@ -20,6 +20,7 @@ func RegisterBootstrap(rt core_runtime.Runtime) error { rt.Config().DpServer.Authn.EnableReloadableTokens, rt.Config().DpServer.Hds.Enabled, rt.Config().GetEnvoyAdminPort(), + rt.Config().Experimental.UseDeltaXDS, ) if err != nil { return err diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 00581593229b..c8fe9a6c3657 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -38,6 +38,7 @@ func NewDefaultBootstrapGenerator( enableReloadableTokens bool, hdsEnabled bool, defaultAdminPort uint32, + deltaXdsEnabled bool, ) (BootstrapGenerator, error) { hostsAndIps, err := hostsAndIPsFromCertFile(dpServerCertFile) if err != nil { @@ -56,6 +57,7 @@ func NewDefaultBootstrapGenerator( hostsAndIps: hostsAndIps, hdsEnabled: hdsEnabled, defaultAdminPort: defaultAdminPort, + deltaXdsEnabled: deltaXdsEnabled, }, nil } @@ -69,6 +71,7 @@ type bootstrapGenerator struct { hostsAndIps SANSet hdsEnabled bool defaultAdminPort uint32 + deltaXdsEnabled bool } func (b *bootstrapGenerator) Generate(ctx context.Context, request types.BootstrapRequest) (proto.Message, KumaDpBootstrap, error) { @@ -114,7 +117,6 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr MetricsCertPath: request.MetricsResources.CertPath, MetricsKeyPath: request.MetricsResources.KeyPath, SystemCaPath: request.SystemCaPath, - UseDelta: request.XDSConfigType == "delta", } setAdminPort := func(adminPortFromResource uint32) { @@ -125,6 +127,12 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr } } + if request.XDSConfigType == "" { + params.UseDelta = b.deltaXdsEnabled + } else { + params.UseDelta = request.XDSConfigType == "delta" + } + switch mesh_proto.ProxyType(params.ProxyType) { case mesh_proto.IngressProxyType: zoneIngress, err := b.zoneIngressFor(ctx, request, proxyId) diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index c33e5c88330d..da2a77d1ea7c 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -117,7 +117,7 @@ var _ = Describe("bootstrapGenerator", func() { proxyConfig = *given.proxyConfig } - generator, err := NewDefaultBootstrapGenerator(resManager, given.serverConfig, proxyConfig, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), given.dpAuthForProxyType, given.useTokenPath, given.hdsEnabled, 0) + generator, err := NewDefaultBootstrapGenerator(resManager, given.serverConfig, proxyConfig, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), given.dpAuthForProxyType, given.useTokenPath, given.hdsEnabled, 0, false) Expect(err).ToNot(HaveOccurred()) // when @@ -205,6 +205,34 @@ var _ = Describe("bootstrapGenerator", func() { expectedConfigFile: "generator.custom-config-minimal-request.golden.yaml", hdsEnabled: true, }), + Entry("custom config with minimal request and delta", testCase{ + dpAuthForProxyType: map[string]bool{}, + serverConfig: func() *bootstrap_config.BootstrapServerConfig { + return &bootstrap_config.BootstrapServerConfig{ + Params: &bootstrap_config.BootstrapParamsConfig{ + AdminAddress: "192.168.0.1", // by default, Envoy Admin interface should listen on loopback address + AdminAccessLogPath: "/var/log", + XdsHost: "localhost", + XdsPort: 15678, + XdsConnectTimeout: config_types.Duration{Duration: 2 * time.Second}, + }, + } + }(), + dataplane: func() *core_mesh.DataplaneResource { + dp := defaultDataplane() + dp.Spec.Networking.Admin.Port = 9902 + return dp + }, + request: types.BootstrapRequest{ + Mesh: "mesh", + Name: "name.namespace", + Version: defaultVersion, + Workdir: "/tmp", + XDSConfigType: "delta", + }, + expectedConfigFile: "generator.custom-config-minimal-request-and-delta.golden.yaml", + hdsEnabled: true, + }), Entry("custom config", testCase{ dpAuthForProxyType: authEnabled, serverConfig: func() *bootstrap_config.BootstrapServerConfig { @@ -518,7 +546,7 @@ var _ = Describe("bootstrapGenerator", func() { cfg := bootstrap_config.DefaultBootstrapServerConfig() proxyCfg := xds_config.DefaultProxyConfig() - generator, err := NewDefaultBootstrapGenerator(resManager, cfg, proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), map[string]bool{}, false, true, 9901) + generator, err := NewDefaultBootstrapGenerator(resManager, cfg, proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), map[string]bool{}, false, true, 9901, false) Expect(err).ToNot(HaveOccurred()) // when @@ -656,7 +684,7 @@ Provide CA that was used to sign a certificate used in the control plane by usin err = resManager.Create(context.Background(), dataplane, store.CreateByKey("name.namespace", "metrics")) Expect(err).ToNot(HaveOccurred()) - generator, err := NewDefaultBootstrapGenerator(resManager, config(), proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, false, 0) + generator, err := NewDefaultBootstrapGenerator(resManager, config(), proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, false, 0, false) Expect(err).ToNot(HaveOccurred()) // when @@ -751,7 +779,7 @@ Provide CA that was used to sign a certificate used in the control plane by usin err = resManager.Create(context.Background(), dataplane, store.CreateByKey("name.namespace", "metrics")) Expect(err).ToNot(HaveOccurred()) - generator, err := NewDefaultBootstrapGenerator(resManager, config(), proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, false, 0) + generator, err := NewDefaultBootstrapGenerator(resManager, config(), proxyCfg, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, false, 0, false) Expect(err).ToNot(HaveOccurred()) // when diff --git a/pkg/xds/bootstrap/server_test.go b/pkg/xds/bootstrap/server_test.go index 68585c16c473..929bfc3a87b6 100644 --- a/pkg/xds/bootstrap/server_test.go +++ b/pkg/xds/bootstrap/server_test.go @@ -95,7 +95,7 @@ var _ = Describe("Bootstrap Server", func() { proxyConfig := xds_config.DefaultProxyConfig() - generator, err := bootstrap.NewDefaultBootstrapGenerator(resManager, config, proxyConfig, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, true, 0) + generator, err := bootstrap.NewDefaultBootstrapGenerator(resManager, config, proxyConfig, filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem"), authEnabled, false, true, 0, false) Expect(err).ToNot(HaveOccurred()) bootstrapHandler := bootstrap.BootstrapHandler{ Generator: generator, diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go index e31ad9f7e2fc..ed3dc201b5e7 100644 --- a/pkg/xds/bootstrap/template_v3.go +++ b/pkg/xds/bootstrap/template_v3.go @@ -106,10 +106,10 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada }, }) } - configType := envoy_core_v3.ApiConfigSource_DELTA_GRPC - // if parameters.UseDelta { - // configType = envoy_core_v3.ApiConfigSource_DELTA_GRPC - // } + configType := envoy_core_v3.ApiConfigSource_GRPC + if parameters.UseDelta { + configType = envoy_core_v3.ApiConfigSource_DELTA_GRPC + } res := &envoy_bootstrap_v3.Bootstrap{ Node: &envoy_core_v3.Node{ @@ -233,7 +233,7 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada } if parameters.HdsEnabled { res.HdsConfig = &envoy_core_v3.ApiConfigSource{ - ApiType: configType, + ApiType: envoy_core_v3.ApiConfigSource_GRPC, TransportApiVersion: envoy_core_v3.ApiVersion_V3, SetNodeOnFirstMessageOnly: true, GrpcServices: []*envoy_core_v3.GrpcService{ diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml new file mode 100644 index 000000000000..d813a9ebc47f --- /dev/null +++ b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml @@ -0,0 +1,145 @@ +admin: + accessLog: + - name: envoy.access_loggers.file + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /var/log + address: + socketAddress: + address: 192.168.0.1 + portValue: 9902 +defaultRegexEngine: + name: envoy.regex_engines.google_re2 + typedConfig: + '@type': type.googleapis.com/envoy.extensions.regex_engines.v3.GoogleRE2 +dynamicResources: + adsConfig: + apiType: DELTA_GRPC + grpcServices: + - envoyGrpc: + clusterName: ads_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 + cdsConfig: + ads: {} + initialFetchTimeout: 0s + resourceApiVersion: V3 + ldsConfig: + ads: {} + initialFetchTimeout: 0s + resourceApiVersion: V3 +hdsConfig: + apiType: GRPC + grpcServices: + - envoyGrpc: + clusterName: ads_cluster + setNodeOnFirstMessageOnly: true + transportApiVersion: V3 +layeredRuntime: + layers: + - name: kuma + staticLayer: + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +node: + cluster: backend + id: mesh.name.namespace + metadata: + dataplane.admin.address: 192.168.0.1 + dataplane.admin.port: "9902" + dataplane.proxyType: dataplane + features: [] + metricsCertPath: "" + metricsKeyPath: "" + systemCaPath: "" + version: + envoy: + build: hash/1.15.0/RELEASE + version: 1.15.0 + kumaDp: + buildDate: "2019-08-07T11:26:06Z" + gitCommit: 91ce236824a9d875601679aa80c63783fb0e8725 + gitTag: v0.0.1 + version: 0.0.1 + workdir: /tmp +staticResources: + clusters: + - connectTimeout: 2s + loadAssignment: + clusterName: access_log_sink + endpoints: + - lbEndpoints: + - endpoint: + address: + pipe: + path: /tmp/kuma-al-name.namespace-mesh.sock + name: access_log_sink + type: STATIC + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} + upstreamConnectionOptions: + tcpKeepalive: + keepaliveInterval: 10 + keepaliveProbes: 3 + keepaliveTime: 10 + - connectTimeout: 2s + loadAssignment: + clusterName: ads_cluster + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: localhost + portValue: 15678 + name: ads_cluster + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + tlsParams: + tlsMinimumProtocolVersion: TLSv1_2 + validationContextSdsSecretConfig: + name: cp_validation_ctx + sni: localhost + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: {} + upstreamConnectionOptions: + tcpKeepalive: + keepaliveInterval: 10 + keepaliveProbes: 3 + keepaliveTime: 10 + secrets: + - name: cp_validation_ctx + validationContext: + matchTypedSubjectAltNames: + - matcher: + exact: localhost + sanType: DNS + - matcher: + exact: localhost + sanType: IP_ADDRESS + trustedCa: + inlineBytes: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURNekNDQWh1Z0F3SUJBZ0lRRGhsSW5mc1hZSGFtS04rMjlxblF2ekFOQmdrcWhraUc5dzBCQVFzRkFEQVAKTVEwd0N3WURWUVFERXdScmRXMWhNQjRYRFRJeE1EUXdNakV3TWpJeU5sb1hEVE14TURNek1URXdNakl5TmxvdwpEekVOTUFzR0ExVUVBeE1FYTNWdFlUQ0NBU0l3RFFZSktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCCkFMNEdHZytlMk83ZUExMkYwRjZ2MnJyOGoyaVZTRktlcG5adEwxNWxyQ2RzNmxxSzUwc1hXT3c4UEtacDJpaEEKWEpWVFNaekthc3lMRFRBUjlWWVFqVHBFNTI2RXp2dGR0aFNhZ2YzMlFXVyt3WTZMTXBFZGV4S09PQ3gyc2U1NQpSZDk3TDMzeVlQZmdYMTVPWWxpSFBEMDU2ampob3RITGROMmxweTcrU1REdlF5Um5YQXU3M1lrWTM3RWQ0aEk0CnQvVjZzb0h5RUdOY0RobTlwNWZCR3F6MG5qQmJRa3AybFRZNS9rajQycUI3UTZyQ00ydGJQc0VNb29lQUF3NW0KaHlZNHhqMHRQOXVjcWxVejhnYys2bzhIRE5zdDhOZUpYWmt0V24rQ095dGpyL056R2dTMjJrdlNEcGhpc0pvdApvMEZ5b0lPZEF0eEMxcXhYWFIrWHVVVUNBd0VBQWFPQmlqQ0JoekFPQmdOVkhROEJBZjhFQkFNQ0FxUXdIUVlEClZSMGxCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQk1BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWUQKVlIwT0JCWUVGS1JMa2dJelgvT2pLdzlpZGVwdVEvUk10VCtBTUNZR0ExVWRFUVFmTUIyQ0NXeHZZMkZzYUc5egpkSWNRL1FDaEl3QUFBQUFBQUFBQUFBQUFBVEFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBUHM1eUpaaG9ZbEdXCkNwQThkU0lTaXZNOC84aUJOUTNmVndQNjNmdDBFSkxNVkd1MlJGWjQvVUFKL3JVUFNHTjh4aFhTazUrMWQ1NmEKL2thSDlyWDBIYVJJSEhseEE3aVBVS3hBajQ0eDlMS21xUEhUb0wzWGxXWTFBWHp2aWNXOWQrR00yRmFRZWUrSQpsZWFxTGJ6MEFadmxudTI3MVoxQ2VhQUN1VTlHbGp1anZ5aVRURTluYUhVRXF2SGdTcFB0aWxKYWx5SjUveklsClo5RjArVVd0M1RPWU1zNWcrU0N0ME13SFROYmlzYm1ld3BjRkZKemp0Mmt2dHJjOXQ5ZGtGODF4aGNTMTl3N3EKaDFBZVAzUlJsTGw3YnY5RUFWWEVtSWF2aWgvMjlQQTNaU3krcGJZTlc3ak5KSGpNUTRoUTBFK3hjQ2F6VS9PNAp5cFdHYWFudlBnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= +statsConfig: + statsTags: + - regex: ^grpc\.((.+)\.) + tagName: name + - regex: ^grpc.*streams_closed(_([0-9]+)) + tagName: status + - regex: ^kafka(\.(\S*[0-9]))\. + tagName: kafka_name + - regex: ^kafka\..*\.(.*?(?=_duration|$)) + tagName: kafka_type + - regex: (worker_([0-9]+)\.) + tagName: worker + - regex: ((.+?)\.)rbac\. + tagName: listener From b9b339b08e2f825609bda85c38cbf6d34141cf2e Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 5 Nov 2024 15:36:28 +0100 Subject: [PATCH 05/12] cleanup Signed-off-by: Lukasz Dziedziak --- .../pkg/dataplane/envoy/remote_bootstrap.go | 6 +---- .../dataplane/envoy/remote_bootstrap_test.go | 1 + .../testdata/bootstrap-request-0.golden.json | 3 +-- .../testdata/bootstrap-request-1.golden.json | 3 +-- .../testdata/bootstrap-request-2.golden.json | 3 +-- .../testdata/bootstrap-request-3.golden.json | 2 +- pkg/config/app/kuma-dp/config.go | 11 ++------- .../testdata/invalid-config.golden.txt | 2 +- .../testdata/invalid-config.input.yaml | 1 - .../kuma-dp/testdata/valid-config.input.yaml | 2 +- pkg/xds/bootstrap/generator.go | 7 +----- pkg/xds/bootstrap/generator_test.go | 10 ++++---- pkg/xds/bootstrap/types/bootstrap_request.go | 24 +++++++++---------- test/framework/envs/multizone/env.go | 1 + test/framework/interface.go | 7 ++++++ test/framework/k8s_cluster.go | 4 ++++ 16 files changed, 40 insertions(+), 47 deletions(-) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index 71c3a508e943..ebd11f70ee2e 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -191,11 +191,7 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. }, SystemCaPath: params.SystemCaPath, } - if cfg.DataplaneRuntime.XDSConfigType == "" { - request.XDSConfigType = "sotw" - } else { - request.XDSConfigType = "delta" - } + request.DeltaXDSConfigEnabled = cfg.DataplaneRuntime.DeltaXdsConfigEnabled jsonBytes, err := json.MarshalIndent(request, "", " ") if err != nil { return nil, errors.Wrap(err, "could not marshal request to json") diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index 1a0882e80a74..f49645ced999 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -149,6 +149,7 @@ var _ = Describe("Remote Bootstrap", func() { cfg.Dataplane.Mesh = "demo" cfg.Dataplane.Name = "sample" cfg.DataplaneRuntime.Token = "token" + cfg.DataplaneRuntime.DeltaXdsConfigEnabled = true return testCase{ config: cfg, diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json index faa20220fbcb..6f63ce578d65 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-0.golden.json @@ -31,6 +31,5 @@ "certPath": "/tmp/cert.pem", "keyPath": "/tmp/key.pem" }, - "systemCaPath": "", - "xdsConfigType": "sotw" + "systemCaPath": "" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json index f5da93dc81d9..5e5d5da29c45 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-1.golden.json @@ -30,6 +30,5 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "", - "xdsConfigType": "sotw" + "systemCaPath": "" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json index ec718bd63d64..473d5360187d 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-2.golden.json @@ -29,6 +29,5 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "", - "xdsConfigType": "sotw" + "systemCaPath": "" } diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json index ec718bd63d64..e6a2923c8a5c 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json @@ -30,5 +30,5 @@ "keyPath": "" }, "systemCaPath": "", - "xdsConfigType": "sotw" + "deltaXdsConfigEnabled": true } diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index adfb20050894..308395b1d3f6 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -217,8 +217,8 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` - // XDSConfigType defines xDS communication type between Envoy and control-plane - XDSConfigType string `json:"xdsConfigType,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_type"` + // DeltaXdsConfigEnabled enables incremental xDS communication type between Envoy and control-plane + DeltaXdsConfigEnabled bool `json:"deltaXdsConfigEnabled,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_delta_xds_config_enabled"` } type Metrics struct { @@ -337,13 +337,6 @@ func (d *DataplaneRuntime) Validate() error { if d.BinaryPath == "" { errs = multierr.Append(errs, errors.Errorf(".BinaryPath must be non-empty")) } - if d.XDSConfigType != "" { - switch d.XDSConfigType { - case "delta", "sotw": - default: - errs = multierr.Append(errs, errors.Errorf(".XDSConfigType can be one of: delta, sotw")) - } - } return errs } diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt index 59e5a9b2bf4a..5ca12650bb24 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt @@ -1 +1 @@ -parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty; .XDSConfigType can be one of: delta, sotw +parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml index f2b1e736c7fb..7467f0431afa 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml @@ -12,4 +12,3 @@ dataplane: proxyType: not-a-proxy dataplaneRuntime: binaryPath: - xdsConfigType: a diff --git a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml index 0ab02544046e..596ee92c0a5b 100644 --- a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml @@ -12,4 +12,4 @@ dataplaneRuntime: binaryPath: envoy.sh configDir: /var/run/envoy envoyLogLevel: trace - xdsConfigType: delta + deltaXdsConfigEnabled: true diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 8503d98c7b04..d6a95269168c 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -127,12 +127,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr params.AdminPort = b.defaultAdminPort } } - - if request.XDSConfigType == "" { - params.UseDelta = b.deltaXdsEnabled - } else { - params.UseDelta = request.XDSConfigType == "delta" - } + params.UseDelta = request.DeltaXDSConfigEnabled switch mesh_proto.ProxyType(params.ProxyType) { case mesh_proto.IngressProxyType: diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index 599137d1cebc..065c871e50cc 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -224,11 +224,11 @@ var _ = Describe("bootstrapGenerator", func() { return dp }, request: types.BootstrapRequest{ - Mesh: "mesh", - Name: "name.namespace", - Version: defaultVersion, - Workdir: "/tmp", - XDSConfigType: "delta", + Mesh: "mesh", + Name: "name.namespace", + Version: defaultVersion, + Workdir: "/tmp", + DeltaXDSConfigEnabled: true, }, expectedConfigFile: "generator.custom-config-minimal-request-and-delta.golden.yaml", hdsEnabled: true, diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index eec6ae85ee5c..0e6354277fc6 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -10,18 +10,18 @@ type BootstrapRequest struct { Host string `json:"-"` Version Version `json:"version"` // CaCert is a PEM-encoded CA cert that DP uses to verify CP - CaCert string `json:"caCert"` - DynamicMetadata map[string]string `json:"dynamicMetadata"` - DNSPort uint32 `json:"dnsPort,omitempty"` - ReadinessPort uint32 `json:"readinessPort,omitempty"` - AppProbeProxyEnabled bool `json:"appProbeProxyDisabled,omitempty"` - OperatingSystem string `json:"operatingSystem"` - Features []string `json:"features"` - Resources ProxyResources `json:"resources"` - Workdir string `json:"workdir"` - MetricsResources MetricsResources `json:"metricsResources"` - SystemCaPath string `json:"systemCaPath"` - XDSConfigType string `json:"xdsConfigType,omitempty"` + CaCert string `json:"caCert"` + DynamicMetadata map[string]string `json:"dynamicMetadata"` + DNSPort uint32 `json:"dnsPort,omitempty"` + ReadinessPort uint32 `json:"readinessPort,omitempty"` + AppProbeProxyEnabled bool `json:"appProbeProxyDisabled,omitempty"` + OperatingSystem string `json:"operatingSystem"` + Features []string `json:"features"` + Resources ProxyResources `json:"resources"` + Workdir string `json:"workdir"` + MetricsResources MetricsResources `json:"metricsResources"` + SystemCaPath string `json:"systemCaPath"` + DeltaXDSConfigEnabled bool `json:"deltaXdsConfigEnabled,omitempty"` } type Version struct { diff --git a/test/framework/envs/multizone/env.go b/test/framework/envs/multizone/env.go index d580fb3e815f..40b5cde15bc4 100644 --- a/test/framework/envs/multizone/env.go +++ b/test/framework/envs/multizone/env.go @@ -132,6 +132,7 @@ func SetupAndGetState() []byte { kubeZone2Options := framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.KubeZone2) kubeZone2Options = append(kubeZone2Options, WithCNI()) + kubeZone2Options = append(kubeZone2Options, WithDeltaXds()) KubeZone2 = setupKubeZone(&wg, Kuma2, kubeZone2Options...) UniZone1 = setupUniZone(&wg, Kuma4, framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.UniZone1)...) diff --git a/test/framework/interface.go b/test/framework/interface.go index 4ccf2f54b139..13bdb9c2e6c9 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -51,6 +51,7 @@ type kumaDeploymentOptions struct { zoneName string verifyKuma bool setupKumactl bool + useDeltaXds bool // Functions to apply to each mesh after the control plane // is provisioned. @@ -321,6 +322,12 @@ func WithCNI() KumaDeploymentOption { }) } +func WithDeltaXds() KumaDeploymentOption { + return KumaOptionFunc(func(o *kumaDeploymentOptions) { + o.useDeltaXds = true + }) +} + func WithCNINamespace(namespace string) KumaDeploymentOption { return KumaOptionFunc(func(o *kumaDeploymentOptions) { o.cniNamespace = namespace diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go index d2972e40e4e6..6a08473fcf45 100644 --- a/test/framework/k8s_cluster.go +++ b/test/framework/k8s_cluster.go @@ -382,6 +382,10 @@ func (c *K8sCluster) yamlForKumaViaKubectl(mode string) (string, error) { argsMap["--cni-conf-name"] = Config.CNIConf.ConfName } + if c.opts.useDeltaXds { + argsMap["--env-var"] = "KUMA_EXPERIMENTAL_USE_DELTA_XDS=true" + } + if Config.XDSApiVersion != "" { argsMap["--env-var"] = "KUMA_BOOTSTRAP_SERVER_API_VERSION=" + Config.XDSApiVersion } From 9ce981f09b681be55d585dec58114b0b36caac37 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 5 Nov 2024 16:38:50 +0100 Subject: [PATCH 06/12] cleanup Signed-off-by: Lukasz Dziedziak --- .../pkg/dataplane/envoy/remote_bootstrap.go | 9 +++++- .../dataplane/envoy/remote_bootstrap_test.go | 2 +- .../testdata/bootstrap-request-3.golden.json | 2 +- pkg/config/app/kuma-cp/config.go | 2 +- pkg/config/app/kuma-dp/config.go | 11 +++++-- .../testdata/invalid-config.golden.txt | 2 +- .../testdata/invalid-config.input.yaml | 1 + .../kuma-dp/testdata/valid-config.input.yaml | 2 +- pkg/config/loader_test.go | 4 +-- pkg/xds/bootstrap/components.go | 2 +- pkg/xds/bootstrap/generator.go | 9 +++++- pkg/xds/bootstrap/generator_test.go | 10 +++--- pkg/xds/bootstrap/types/bootstrap_request.go | 32 ++++++++++++------- test/framework/envs/multizone/env.go | 6 ++-- test/framework/interface.go | 7 ---- test/framework/k8s_cluster.go | 4 --- 16 files changed, 63 insertions(+), 42 deletions(-) diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index ebd11f70ee2e..e837020e43a8 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -191,7 +191,14 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. }, SystemCaPath: params.SystemCaPath, } - request.DeltaXDSConfigEnabled = cfg.DataplaneRuntime.DeltaXdsConfigEnabled + switch cfg.DataplaneRuntime.XdsConfigMode { + case "delta": + request.XdsConfigMode = types.DELTA + case "sotw": + request.XdsConfigMode = types.SOTW + default: + request.XdsConfigMode = types.NOT_DEFINED + } jsonBytes, err := json.MarshalIndent(request, "", " ") if err != nil { return nil, errors.Wrap(err, "could not marshal request to json") diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index f49645ced999..08f9e9ff5360 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -149,7 +149,7 @@ var _ = Describe("Remote Bootstrap", func() { cfg.Dataplane.Mesh = "demo" cfg.Dataplane.Name = "sample" cfg.DataplaneRuntime.Token = "token" - cfg.DataplaneRuntime.DeltaXdsConfigEnabled = true + cfg.DataplaneRuntime.XdsConfigMode = "delta" return testCase{ config: cfg, diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json index e6a2923c8a5c..15328d0acc5e 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json @@ -30,5 +30,5 @@ "keyPath": "" }, "systemCaPath": "", - "deltaXdsConfigEnabled": true + "xdsConfigMode": 2 } diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index 0768f4f67cba..4529711c7209 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -465,7 +465,7 @@ type ExperimentalConfig struct { // Do not enable on production. SkipPersistedVIPs bool `json:"skipPersistedVIPs" envconfig:"KUMA_EXPERIMENTAL_SKIP_PERSISTED_VIPS"` // If true uses Delta xDS to deliver changes to sidecars. - UseDeltaXDS bool `json:"useDeltaXDS" envconfig:"KUMA_EXPERIMENTAL_USE_DELTA_XDS"` + UseDeltaXds bool `json:"useDeltaXds" envconfig:"KUMA_EXPERIMENTAL_USE_DELTA_XDS"` } type ExperimentalKDSEventBasedWatchdog struct { diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index 308395b1d3f6..9dd1e927a9b9 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -217,8 +217,8 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` - // DeltaXdsConfigEnabled enables incremental xDS communication type between Envoy and control-plane - DeltaXdsConfigEnabled bool `json:"deltaXdsConfigEnabled,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_delta_xds_config_enabled"` + // XdsConfigMode defines xDS communication type between Envoy and control-plane + XdsConfigMode string `json:"xdsConfigMode,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_mode"` } type Metrics struct { @@ -337,6 +337,13 @@ func (d *DataplaneRuntime) Validate() error { if d.BinaryPath == "" { errs = multierr.Append(errs, errors.Errorf(".BinaryPath must be non-empty")) } + if d.XdsConfigMode != "" { + switch d.XdsConfigMode { + case "delta", "sotw": + default: + errs = multierr.Append(errs, errors.Errorf(".XdsConfigMode can be one of: delta, sotw")) + } + } return errs } diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt index 5ca12650bb24..fedc368f1e63 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt @@ -1 +1 @@ -parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty +parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty; .XdsConfigMode can be one of: delta, sotw diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml index 7467f0431afa..daf21d15719b 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml @@ -12,3 +12,4 @@ dataplane: proxyType: not-a-proxy dataplaneRuntime: binaryPath: + xdsConfigMode: a diff --git a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml index 596ee92c0a5b..e79cb0c531d2 100644 --- a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml @@ -12,4 +12,4 @@ dataplaneRuntime: binaryPath: envoy.sh configDir: /var/run/envoy envoyLogLevel: trace - deltaXdsConfigEnabled: true + xdsConfigMode: delta diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 0005f361a231..26adac35258f 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -372,7 +372,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Experimental.SidecarContainers).To(BeTrue()) Expect(cfg.Experimental.SkipPersistedVIPs).To(BeTrue()) Expect(cfg.Experimental.GenerateMeshServices).To(BeTrue()) - Expect(cfg.Experimental.UseDeltaXDS).To(BeTrue()) + Expect(cfg.Experimental.UseDeltaXds).To(BeTrue()) Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1)) Expect(cfg.EventBus.BufferSize).To(Equal(uint(30))) @@ -760,7 +760,7 @@ experimental: sidecarContainers: true generateMeshServices: true skipPersistedVIPs: true - useDeltaXDS: true + useDeltaXds: true proxy: gateway: globalDownstreamMaxConnections: 1 diff --git a/pkg/xds/bootstrap/components.go b/pkg/xds/bootstrap/components.go index e657cbeccefa..7d793920f078 100644 --- a/pkg/xds/bootstrap/components.go +++ b/pkg/xds/bootstrap/components.go @@ -21,7 +21,7 @@ func RegisterBootstrap(rt core_runtime.Runtime) error { rt.Config().DpServer.Authn.EnableReloadableTokens || rt.Config().Store.Type == store.KubernetesStore, rt.Config().DpServer.Hds.Enabled, rt.Config().GetEnvoyAdminPort(), - rt.Config().Experimental.UseDeltaXDS, + rt.Config().Experimental.UseDeltaXds, ) if err != nil { return err diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index d6a95269168c..60264e6b1c8e 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -127,7 +127,14 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr params.AdminPort = b.defaultAdminPort } } - params.UseDelta = request.DeltaXDSConfigEnabled + switch request.XdsConfigMode { + case types.DELTA: + params.UseDelta = true + case types.NOT_DEFINED: + if b.deltaXdsEnabled { + params.UseDelta = true + } + } switch mesh_proto.ProxyType(params.ProxyType) { case mesh_proto.IngressProxyType: diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index 065c871e50cc..1141fbb25086 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -224,11 +224,11 @@ var _ = Describe("bootstrapGenerator", func() { return dp }, request: types.BootstrapRequest{ - Mesh: "mesh", - Name: "name.namespace", - Version: defaultVersion, - Workdir: "/tmp", - DeltaXDSConfigEnabled: true, + Mesh: "mesh", + Name: "name.namespace", + Version: defaultVersion, + Workdir: "/tmp", + XdsConfigMode: types.DELTA, }, expectedConfigFile: "generator.custom-config-minimal-request-and-delta.golden.yaml", hdsEnabled: true, diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index 0e6354277fc6..6fbdb094d9ab 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -10,20 +10,28 @@ type BootstrapRequest struct { Host string `json:"-"` Version Version `json:"version"` // CaCert is a PEM-encoded CA cert that DP uses to verify CP - CaCert string `json:"caCert"` - DynamicMetadata map[string]string `json:"dynamicMetadata"` - DNSPort uint32 `json:"dnsPort,omitempty"` - ReadinessPort uint32 `json:"readinessPort,omitempty"` - AppProbeProxyEnabled bool `json:"appProbeProxyDisabled,omitempty"` - OperatingSystem string `json:"operatingSystem"` - Features []string `json:"features"` - Resources ProxyResources `json:"resources"` - Workdir string `json:"workdir"` - MetricsResources MetricsResources `json:"metricsResources"` - SystemCaPath string `json:"systemCaPath"` - DeltaXDSConfigEnabled bool `json:"deltaXdsConfigEnabled,omitempty"` + CaCert string `json:"caCert"` + DynamicMetadata map[string]string `json:"dynamicMetadata"` + DNSPort uint32 `json:"dnsPort,omitempty"` + ReadinessPort uint32 `json:"readinessPort,omitempty"` + AppProbeProxyEnabled bool `json:"appProbeProxyDisabled,omitempty"` + OperatingSystem string `json:"operatingSystem"` + Features []string `json:"features"` + Resources ProxyResources `json:"resources"` + Workdir string `json:"workdir"` + MetricsResources MetricsResources `json:"metricsResources"` + SystemCaPath string `json:"systemCaPath"` + XdsConfigMode XdsMode `json:"xdsConfigMode,omitempty"` } +type XdsMode int + +const ( + NOT_DEFINED XdsMode = iota + SOTW + DELTA +) + type Version struct { KumaDp KumaDpVersion `json:"kumaDp"` Envoy EnvoyVersion `json:"envoy"` diff --git a/test/framework/envs/multizone/env.go b/test/framework/envs/multizone/env.go index 40b5cde15bc4..a1dada76f876 100644 --- a/test/framework/envs/multizone/env.go +++ b/test/framework/envs/multizone/env.go @@ -130,9 +130,11 @@ func SetupAndGetState() []byte { } KubeZone1 = setupKubeZone(&wg, Kuma1, kubeZone1Options...) - kubeZone2Options := framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.KubeZone2) + kubeZone2Options := append( + framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.KubeZone2), + WithEnv("KUMA_EXPERIMENTAL_USE_DELTA_XDS", "true"), + ) kubeZone2Options = append(kubeZone2Options, WithCNI()) - kubeZone2Options = append(kubeZone2Options, WithDeltaXds()) KubeZone2 = setupKubeZone(&wg, Kuma2, kubeZone2Options...) UniZone1 = setupUniZone(&wg, Kuma4, framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.UniZone1)...) diff --git a/test/framework/interface.go b/test/framework/interface.go index 13bdb9c2e6c9..4ccf2f54b139 100644 --- a/test/framework/interface.go +++ b/test/framework/interface.go @@ -51,7 +51,6 @@ type kumaDeploymentOptions struct { zoneName string verifyKuma bool setupKumactl bool - useDeltaXds bool // Functions to apply to each mesh after the control plane // is provisioned. @@ -322,12 +321,6 @@ func WithCNI() KumaDeploymentOption { }) } -func WithDeltaXds() KumaDeploymentOption { - return KumaOptionFunc(func(o *kumaDeploymentOptions) { - o.useDeltaXds = true - }) -} - func WithCNINamespace(namespace string) KumaDeploymentOption { return KumaOptionFunc(func(o *kumaDeploymentOptions) { o.cniNamespace = namespace diff --git a/test/framework/k8s_cluster.go b/test/framework/k8s_cluster.go index 6a08473fcf45..d2972e40e4e6 100644 --- a/test/framework/k8s_cluster.go +++ b/test/framework/k8s_cluster.go @@ -382,10 +382,6 @@ func (c *K8sCluster) yamlForKumaViaKubectl(mode string) (string, error) { argsMap["--cni-conf-name"] = Config.CNIConf.ConfName } - if c.opts.useDeltaXds { - argsMap["--env-var"] = "KUMA_EXPERIMENTAL_USE_DELTA_XDS=true" - } - if Config.XDSApiVersion != "" { argsMap["--env-var"] = "KUMA_BOOTSTRAP_SERVER_API_VERSION=" + Config.XDSApiVersion } From f1fcb25ca7ce15222ac52410d5f9090d4439ec14 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 5 Nov 2024 16:45:38 +0100 Subject: [PATCH 07/12] trigger build Signed-off-by: Lukasz Dziedziak From 5e99aef276e3bfc13c9566014afaa14d3e3d5904 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Wed, 6 Nov 2024 10:57:23 +0100 Subject: [PATCH 08/12] added better doc Signed-off-by: Lukasz Dziedziak --- pkg/config/app/kuma-dp/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index 9dd1e927a9b9..bd0e5b360f9a 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -217,7 +217,9 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` - // XdsConfigMode defines xDS communication type between Envoy and control-plane + // XdsConfigMode defines xDS communication type between Envoy and control-plane. + // Available values are: sotw, delta + // By default it takes configuration provided by the control-plane. XdsConfigMode string `json:"xdsConfigMode,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_mode"` } From 811c7beb6578e928e7a345e0e891e44d50c94ac6 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Fri, 8 Nov 2024 17:36:19 +0100 Subject: [PATCH 09/12] fix k8s by adding new field in dataplane Signed-off-by: Lukasz Dziedziak --- api/mesh/v1alpha1/dataplane.pb.go | 622 +++++++++--------- api/mesh/v1alpha1/dataplane.proto | 4 + api/mesh/v1alpha1/envoy.pb.go | 215 ++++++ api/mesh/v1alpha1/envoy.proto | 19 + api/mesh/v1alpha1/zone_ingress.pb.go | 158 +++-- api/mesh/v1alpha1/zone_ingress.proto | 4 + api/mesh/v1alpha1/zoneegress.pb.go | 100 +-- api/mesh/v1alpha1/zoneegress.proto | 4 + .../pkg/dataplane/envoy/remote_bootstrap.go | 8 - .../dataplane/envoy/remote_bootstrap_test.go | 1 - .../testdata/bootstrap-request-3.golden.json | 3 +- docs/generated/raw/protos/Dataplane.json | 31 + .../raw/protos/DataplaneOverview.json | 31 + .../raw/protos/EnvoyConfiguration.json | 32 + docs/generated/raw/protos/ZoneEgress.json | 31 + .../raw/protos/ZoneEgressOverview.json | 31 + docs/generated/raw/protos/ZoneIngress.json | 31 + .../raw/protos/ZoneIngressOverview.json | 31 + pkg/config/app/kuma-cp/config.go | 2 +- pkg/config/app/kuma-dp/config.go | 11 - .../testdata/invalid-config.golden.txt | 2 +- .../testdata/invalid-config.input.yaml | 1 - .../kuma-dp/testdata/valid-config.input.yaml | 1 - pkg/config/loader_test.go | 6 +- .../k8s/controllers/egress_converter.go | 1 + .../k8s/controllers/envoy_configuration.go | 25 + .../k8s/controllers/ingress_converter.go | 1 + .../k8s/controllers/pod_controller_test.go | 11 + .../runtime/k8s/controllers/pod_converter.go | 3 +- .../controllers/testdata/01.dataplane.yaml | 2 + .../controllers/testdata/02.dataplane.yaml | 2 + .../controllers/testdata/03.dataplane.yaml | 2 + .../controllers/testdata/04.dataplane.yaml | 2 + .../controllers/testdata/05.dataplane.yaml | 2 + .../controllers/testdata/06.dataplane.yaml | 2 + .../controllers/testdata/07.dataplane.yaml | 2 + .../controllers/testdata/08.dataplane.yaml | 2 + .../controllers/testdata/09.dataplane.yaml | 2 + .../controllers/testdata/10.dataplane.yaml | 2 + .../controllers/testdata/11.dataplane.yaml | 2 + .../controllers/testdata/12.dataplane.yaml | 2 + .../controllers/testdata/13.dataplane.yaml | 2 + .../controllers/testdata/14.dataplane.yaml | 2 + .../controllers/testdata/15.dataplane.yaml | 2 + .../controllers/testdata/16.dataplane.yaml | 2 + .../controllers/testdata/17.dataplane.yaml | 2 + .../controllers/testdata/18.dataplane.yaml | 2 + .../controllers/testdata/19.dataplane.yaml | 2 + .../controllers/testdata/20.dataplane.yaml | 2 + .../controllers/testdata/21.dataplane.yaml | 2 + .../controllers/testdata/22.dataplane.yaml | 2 + .../controllers/testdata/23.dataplane.yaml | 2 + .../controllers/testdata/24.dataplane.yaml | 2 + .../controllers/testdata/25.dataplane.yaml | 2 + .../controllers/testdata/26.dataplane.yaml | 2 + .../controllers/testdata/27.dataplane.yaml | 2 + .../controllers/testdata/28.dataplane.yaml | 2 + .../controllers/testdata/29.dataplane.yaml | 2 + .../controllers/testdata/30.dataplane.yaml | 2 + .../k8s/controllers/testdata/30.pod.yaml | 1 + .../testdata/egress/01.dataplane.yaml | 2 + .../testdata/egress/02.dataplane.yaml | 2 + .../testdata/egress/03.dataplane.yaml | 2 + .../testdata/egress/04.dataplane.yaml | 2 + .../testdata/egress/05.dataplane.yaml | 2 + .../testdata/ingress/01.dataplane.yaml | 2 + .../testdata/ingress/02.dataplane.yaml | 2 + .../testdata/ingress/03.dataplane.yaml | 2 + .../testdata/ingress/04.dataplane.yaml | 2 + .../testdata/ingress/05.dataplane.yaml | 2 + .../testdata/ingress/06.dataplane.yaml | 2 + .../ingress/ingress-exists.dataplane.yaml | 2 + .../testdata/mismatch-ports.dataplane.yaml | 2 + .../runtime/k8s/metadata/annotations.go | 4 + pkg/plugins/runtime/k8s/plugin.go | 1 + pkg/xds/bootstrap/components.go | 2 +- pkg/xds/bootstrap/generator.go | 20 +- pkg/xds/bootstrap/generator_test.go | 12 +- pkg/xds/bootstrap/parameters.go | 54 +- pkg/xds/bootstrap/template_v3.go | 4 +- ...nfig-minimal-request-and-delta.golden.yaml | 2 +- .../generator.custom-config.golden.yaml | 3 + pkg/xds/bootstrap/types/bootstrap_request.go | 9 +- test/framework/envs/multizone/env.go | 2 +- 84 files changed, 1130 insertions(+), 490 deletions(-) create mode 100644 api/mesh/v1alpha1/envoy.pb.go create mode 100644 api/mesh/v1alpha1/envoy.proto create mode 100644 docs/generated/raw/protos/EnvoyConfiguration.json create mode 100644 pkg/plugins/runtime/k8s/controllers/envoy_configuration.go diff --git a/api/mesh/v1alpha1/dataplane.pb.go b/api/mesh/v1alpha1/dataplane.pb.go index 89ea8ba4d969..0a8d06a56514 100644 --- a/api/mesh/v1alpha1/dataplane.pb.go +++ b/api/mesh/v1alpha1/dataplane.pb.go @@ -214,6 +214,8 @@ type Dataplane struct { // Deprecated: this feature will be removed for Universal; on Kubernetes, it's // not needed anymore. Probes *Dataplane_Probes `protobuf:"bytes,3,opt,name=probes,proto3" json:"probes,omitempty"` + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + Envoy *EnvoyConfiguration `protobuf:"bytes,4,opt,name=envoy,proto3" json:"envoy,omitempty"` } func (x *Dataplane) Reset() { @@ -269,6 +271,13 @@ func (x *Dataplane) GetProbes() *Dataplane_Probes { return nil } +func (x *Dataplane) GetEnvoy() *EnvoyConfiguration { + if x != nil { + return x.Envoy + } + return nil +} + // Networking describes inbound and outbound interfaces of a data plane proxy. type Dataplane_Networking struct { state protoimpl.MessageState @@ -1338,298 +1347,304 @@ var file_api_mesh_v1alpha1_dataplane_proto_rawDesc = []byte{ 0x61, 0x31, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x23, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, - 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa0, - 0x23, 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x48, 0x0a, 0x0a, - 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x28, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1d, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, + 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xde, 0x23, + 0x0a, 0x09, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x48, 0x0a, 0x0a, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x28, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x3c, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, + 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, + 0x69, 0x63, 0x73, 0x12, 0x3c, 0x0a, 0x06, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, + 0x6e, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x52, 0x06, 0x70, 0x72, 0x6f, 0x62, 0x65, + 0x73, 0x12, 0x3c, 0x0a, 0x05, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x26, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x1a, + 0x8b, 0x1a, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x18, + 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, + 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x4a, 0x0a, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, + 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x52, 0x07, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x12, 0x4a, 0x0a, 0x07, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, + 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x07, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x4d, + 0x0a, 0x08, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x31, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x3c, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, - 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x4d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x07, 0x6d, 0x65, 0x74, - 0x72, 0x69, 0x63, 0x73, 0x12, 0x3c, 0x0a, 0x06, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, - 0x61, 0x6e, 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x52, 0x06, 0x70, 0x72, 0x6f, 0x62, - 0x65, 0x73, 0x1a, 0x8b, 0x1a, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, - 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x61, - 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, - 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x4a, 0x0a, 0x07, 0x67, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, - 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x52, 0x07, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x4a, 0x0a, 0x07, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x52, 0x08, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x6f, 0x0a, + 0x14, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x72, 0x6f, + 0x78, 0x79, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6b, 0x75, + 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x52, 0x13, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x12, 0x34, + 0x0a, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x52, 0x05, 0x61, + 0x64, 0x6d, 0x69, 0x6e, 0x1a, 0xec, 0x07, 0x0a, 0x07, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x70, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, + 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x58, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, - 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x07, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x12, 0x4d, 0x0a, 0x08, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, - 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, - 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x08, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x12, 0x6f, 0x0a, 0x14, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, - 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x52, 0x13, 0x74, 0x72, - 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, - 0x67, 0x12, 0x34, 0x0a, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1e, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, - 0x52, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x1a, 0xec, 0x07, 0x0a, 0x07, 0x49, 0x6e, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x58, 0x0a, 0x04, 0x74, - 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, + 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x9a, 0x01, 0x02, 0x08, 0x01, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x12, 0x4f, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, + 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x06, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x12, 0x61, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, + 0x6f, 0x62, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x54, 0x61, 0x67, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x9a, 0x01, 0x02, 0x08, 0x01, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x4f, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, + 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, - 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x52, 0x06, - 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x61, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6b, - 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x52, 0x0c, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, - 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, - 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, - 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x1a, 0x37, 0x0a, 0x09, 0x54, - 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x1e, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x14, - 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, - 0x65, 0x61, 0x64, 0x79, 0x1a, 0xf0, 0x02, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x50, 0x72, 0x6f, 0x62, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x33, 0x0a, 0x07, - 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x12, 0x4d, 0x0a, 0x13, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x74, - 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x75, 0x6e, - 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, - 0x12, 0x49, 0x0a, 0x11, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x74, 0x68, 0x72, 0x65, - 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, - 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x68, 0x65, 0x61, 0x6c, 0x74, - 0x68, 0x79, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x53, 0x0a, 0x03, 0x74, - 0x63, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, - 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, - 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, - 0x6e, 0x67, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x2e, 0x54, 0x63, 0x70, 0x52, 0x03, 0x74, 0x63, 0x70, - 0x1a, 0x05, 0x0a, 0x03, 0x54, 0x63, 0x70, 0x22, 0x2d, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x09, 0x0a, 0x05, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4e, - 0x6f, 0x74, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x67, 0x6e, - 0x6f, 0x72, 0x65, 0x64, 0x10, 0x02, 0x1a, 0x88, 0x04, 0x0a, 0x08, 0x4f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, - 0x74, 0x12, 0x4f, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x3b, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x12, 0x5c, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, - 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, - 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, - 0x64, 0x52, 0x65, 0x66, 0x52, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, - 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0xe5, 0x01, 0x0a, 0x0a, 0x42, 0x61, - 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, - 0x70, 0x6f, 0x72, 0x74, 0x12, 0x60, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x48, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, - 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, - 0x65, 0x66, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x1a, 0x99, 0x02, 0x0a, 0x07, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x58, 0x0a, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, - 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x61, - 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x9a, 0x01, 0x02, 0x08, - 0x01, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x50, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, + 0x01, 0x1a, 0x1e, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, + 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x1a, 0xf0, 0x02, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, + 0x62, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x33, 0x0a, 0x07, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x4d, + 0x0a, 0x13, 0x75, 0x6e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x74, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, + 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x75, 0x6e, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x79, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x49, 0x0a, + 0x11, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, + 0x6c, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, + 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x79, 0x54, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x53, 0x0a, 0x03, 0x74, 0x63, 0x70, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, - 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x29, 0x0a, 0x0b, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, - 0x12, 0x0b, 0x0a, 0x07, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x01, 0x1a, 0x8a, 0x08, - 0x0a, 0x13, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, - 0x78, 0x79, 0x69, 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x15, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, - 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0d, 0x42, 0x09, 0xfa, 0x42, 0x06, 0x2a, 0x04, 0x18, 0xff, 0xff, 0x03, 0x52, - 0x13, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x3f, 0x0a, 0x16, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0d, 0x42, 0x09, 0xfa, 0x42, 0x06, 0x2a, 0x04, 0x18, 0xff, 0xff, 0x03, 0x52, - 0x14, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, - 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x41, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x72, - 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, - 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x6f, 0x0a, 0x0e, 0x69, 0x70, - 0x5f, 0x66, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x49, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, + 0x72, 0x6f, 0x62, 0x65, 0x2e, 0x54, 0x63, 0x70, 0x52, 0x03, 0x74, 0x63, 0x70, 0x1a, 0x05, 0x0a, + 0x03, 0x54, 0x63, 0x70, 0x22, 0x2d, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x09, 0x0a, + 0x05, 0x52, 0x65, 0x61, 0x64, 0x79, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x4e, 0x6f, 0x74, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x67, 0x6e, 0x6f, 0x72, 0x65, + 0x64, 0x10, 0x02, 0x1a, 0x88, 0x04, 0x0a, 0x08, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x4f, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6b, + 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, + 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, + 0x5c, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, + 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, + 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, + 0x66, 0x52, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x1a, 0x37, 0x0a, + 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0xe5, 0x01, 0x0a, 0x0a, 0x42, 0x61, 0x63, 0x6b, 0x65, + 0x6e, 0x64, 0x52, 0x65, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, + 0x74, 0x12, 0x60, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x48, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, + 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x4f, 0x75, 0x74, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x2e, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x99, + 0x02, 0x0a, 0x07, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x12, 0x58, 0x0a, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, + 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x9a, 0x01, 0x02, 0x08, 0x01, 0x52, 0x04, + 0x74, 0x61, 0x67, 0x73, 0x12, 0x50, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x3c, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, - 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, - 0x2e, 0x49, 0x70, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x0c, 0x69, - 0x70, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x7d, 0x0a, 0x12, 0x72, - 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, - 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, + 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x79, 0x70, 0x65, + 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x29, 0x0a, 0x0b, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, + 0x0a, 0x09, 0x44, 0x45, 0x4c, 0x45, 0x47, 0x41, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, + 0x07, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x01, 0x1a, 0x8a, 0x08, 0x0a, 0x13, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, + 0x6e, 0x67, 0x12, 0x3d, 0x0a, 0x15, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x70, + 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0d, 0x42, 0x09, 0xfa, 0x42, 0x06, 0x2a, 0x04, 0x18, 0xff, 0xff, 0x03, 0x52, 0x13, 0x72, 0x65, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x12, 0x3f, 0x0a, 0x16, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x6f, + 0x72, 0x74, 0x5f, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x42, 0x09, 0xfa, 0x42, 0x06, 0x2a, 0x04, 0x18, 0xff, 0xff, 0x03, 0x52, 0x14, 0x72, 0x65, + 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x50, 0x6f, 0x72, 0x74, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, + 0x6e, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x14, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, 0x61, 0x63, + 0x68, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x05, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x6f, 0x0a, 0x0e, 0x69, 0x70, 0x5f, 0x66, 0x61, + 0x6d, 0x69, 0x6c, 0x79, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x49, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, + 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x2e, 0x49, 0x70, + 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x52, 0x0c, 0x69, 0x70, 0x46, 0x61, + 0x6d, 0x69, 0x6c, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x7d, 0x0a, 0x12, 0x72, 0x65, 0x61, 0x63, + 0x68, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, + 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, + 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, + 0x65, 0x6e, 0x64, 0x73, 0x52, 0x11, 0x72, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x1a, 0xbe, 0x02, 0x0a, 0x13, 0x52, 0x65, 0x61, 0x63, + 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x12, + 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6b, + 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x74, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x5c, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, - 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x52, 0x11, 0x72, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, - 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x1a, 0xbe, 0x02, 0x0a, 0x13, 0x52, - 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, - 0x65, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x74, 0x0a, 0x06, 0x6c, 0x61, - 0x62, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x5c, 0x2e, 0x6b, 0x75, 0x6d, - 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, - 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x2e, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, - 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x79, 0x0a, 0x11, 0x52, - 0x65, 0x61, 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, - 0x12, 0x64, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, - 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x61, - 0x63, 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, - 0x52, 0x04, 0x72, 0x65, 0x66, 0x73, 0x22, 0x42, 0x0a, 0x0c, 0x49, 0x70, 0x46, 0x61, 0x6d, 0x69, - 0x6c, 0x79, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x6e, 0x53, 0x70, 0x65, 0x63, - 0x69, 0x66, 0x69, 0x65, 0x64, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x75, 0x61, 0x6c, 0x53, - 0x74, 0x61, 0x63, 0x6b, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x10, 0x02, - 0x12, 0x08, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x36, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, - 0x52, 0x18, 0x72, 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, - 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x76, 0x36, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, - 0x1a, 0xcf, 0x01, 0x0a, 0x06, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, - 0x4b, 0x0a, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, - 0x65, 0x2e, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x52, 0x09, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x1a, 0x64, 0x0a, 0x08, - 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, - 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x69, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, - 0x74, 0x68, 0x3a, 0xec, 0x05, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x13, 0x0a, 0x11, 0x44, 0x61, 0x74, - 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xaa, 0x8c, - 0x89, 0xa6, 0x01, 0x0b, 0x12, 0x09, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0xaa, - 0x8c, 0x89, 0xa6, 0x01, 0x06, 0x22, 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, 0x8c, 0x89, 0xa6, 0x01, - 0x04, 0x52, 0x02, 0x08, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0d, 0x3a, 0x0b, 0x0a, 0x09, 0x64, - 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, - 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x03, 0x90, 0x01, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x95, 0x01, - 0x8a, 0x01, 0x91, 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, - 0x22, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x66, 0x69, 0x72, 0x73, 0x74, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, - 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5b, 0x30, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, - 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, - 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x96, 0x01, 0x8a, 0x01, 0x92, 0x01, - 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, - 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, + 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, + 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x79, 0x0a, 0x11, 0x52, 0x65, 0x61, 0x63, + 0x68, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x73, 0x12, 0x64, 0x0a, + 0x04, 0x72, 0x65, 0x66, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, 0x2e, 0x6b, 0x75, + 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x61, 0x63, 0x68, 0x61, + 0x62, 0x6c, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x66, 0x52, 0x04, 0x72, + 0x65, 0x66, 0x73, 0x22, 0x42, 0x0a, 0x0c, 0x49, 0x70, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x4d, + 0x6f, 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x6e, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, + 0x65, 0x64, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x75, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x63, + 0x6b, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x50, 0x76, 0x34, 0x10, 0x02, 0x12, 0x08, 0x0a, + 0x04, 0x49, 0x50, 0x76, 0x36, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x52, 0x18, 0x72, + 0x65, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x76, 0x36, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x1a, 0xcf, 0x01, + 0x0a, 0x06, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x4b, 0x0a, 0x09, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2d, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x2e, 0x50, + 0x72, 0x6f, 0x62, 0x65, 0x73, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x09, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73, 0x1a, 0x64, 0x0a, 0x08, 0x45, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, + 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x69, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x3a, + 0xec, 0x05, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x13, 0x0a, 0x11, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, + 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, + 0x0b, 0x12, 0x09, 0x44, 0x61, 0x74, 0x61, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0xaa, 0x8c, 0x89, 0xa6, + 0x01, 0x06, 0x22, 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, + 0x08, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0d, 0x3a, 0x0b, 0x0a, 0x09, 0x64, 0x61, 0x74, 0x61, + 0x70, 0x6c, 0x61, 0x6e, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, 0xaa, 0x8c, 0x89, + 0xa6, 0x01, 0x03, 0x90, 0x01, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x95, 0x01, 0x8a, 0x01, 0x91, + 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x66, 0x69, 0x72, 0x73, 0x74, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x5b, 0x31, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, 0x61, + 0x6e, 0x64, 0x5b, 0x30, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0xa0, 0x01, 0x8a, 0x01, 0x9c, 0x01, 0x64, 0x65, 0x73, + 0x6e, 0x67, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x96, 0x01, 0x8a, 0x01, 0x92, 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74, 0x68, 0x69, - 0x72, 0x64, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, - 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5b, 0x32, - 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, 0x6f, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, - 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x2c, 0x70, - 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x3d, 0x31, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0xa1, 0x01, - 0x8a, 0x01, 0x9d, 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, - 0x22, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, - 0x74, 0x68, 0x65, 0x20, 0x66, 0x6f, 0x75, 0x72, 0x74, 0x68, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, - 0x70, 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5b, 0x33, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, - 0x6b, 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x27, 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, - 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, - 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x2c, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x3d, - 0x31, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, - 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, + 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5b, + 0x31, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, + 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, + 0x65, 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0xaa, + 0x8c, 0x89, 0xa6, 0x01, 0xa0, 0x01, 0x8a, 0x01, 0x9c, 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, + 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74, 0x68, 0x69, 0x72, 0x64, 0x20, + 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, + 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5b, 0x32, 0x5d, 0x2e, 0x74, + 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, 0x61, 0x5c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, 0x5d, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6b, + 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x2c, + 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x2c, 0x70, 0x72, 0x69, 0x6f, + 0x72, 0x69, 0x74, 0x79, 0x3d, 0x31, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0xa1, 0x01, 0x8a, 0x01, 0x9d, + 0x01, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x20, 0x74, 0x61, 0x67, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x65, + 0x20, 0x66, 0x6f, 0x75, 0x72, 0x74, 0x68, 0x20, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x22, + 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, + 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5b, 0x33, 0x5d, 0x2e, 0x74, 0x61, 0x67, 0x73, 0x5b, 0x27, 0x6b, 0x75, 0x6d, + 0x61, 0x5c, 0x2e, 0x69, 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x27, 0x5d, 0x60, + 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x69, 0x6f, 0x2f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x2c, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x3d, 0x31, 0x42, 0x2a, + 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, + 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, + 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -1670,44 +1685,46 @@ var file_api_mesh_v1alpha1_dataplane_proto_goTypes = []interface{}{ nil, // 20: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.LabelsEntry (*Dataplane_Probes_Endpoint)(nil), // 21: kuma.mesh.v1alpha1.Dataplane.Probes.Endpoint (*MetricsBackend)(nil), // 22: kuma.mesh.v1alpha1.MetricsBackend - (*EnvoyAdmin)(nil), // 23: kuma.mesh.v1alpha1.EnvoyAdmin - (*durationpb.Duration)(nil), // 24: google.protobuf.Duration - (*wrapperspb.UInt32Value)(nil), // 25: google.protobuf.UInt32Value + (*EnvoyConfiguration)(nil), // 23: kuma.mesh.v1alpha1.EnvoyConfiguration + (*EnvoyAdmin)(nil), // 24: kuma.mesh.v1alpha1.EnvoyAdmin + (*durationpb.Duration)(nil), // 25: google.protobuf.Duration + (*wrapperspb.UInt32Value)(nil), // 26: google.protobuf.UInt32Value } var file_api_mesh_v1alpha1_dataplane_proto_depIdxs = []int32{ 4, // 0: kuma.mesh.v1alpha1.Dataplane.networking:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking 22, // 1: kuma.mesh.v1alpha1.Dataplane.metrics:type_name -> kuma.mesh.v1alpha1.MetricsBackend 5, // 2: kuma.mesh.v1alpha1.Dataplane.probes:type_name -> kuma.mesh.v1alpha1.Dataplane.Probes - 8, // 3: kuma.mesh.v1alpha1.Dataplane.Networking.gateway:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway - 6, // 4: kuma.mesh.v1alpha1.Dataplane.Networking.inbound:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound - 7, // 5: kuma.mesh.v1alpha1.Dataplane.Networking.outbound:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound - 9, // 6: kuma.mesh.v1alpha1.Dataplane.Networking.transparent_proxying:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying - 23, // 7: kuma.mesh.v1alpha1.Dataplane.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin - 21, // 8: kuma.mesh.v1alpha1.Dataplane.Probes.endpoints:type_name -> kuma.mesh.v1alpha1.Dataplane.Probes.Endpoint - 10, // 9: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.TagsEntry - 11, // 10: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.health:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.Health - 12, // 11: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.serviceProbe:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe - 0, // 12: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.state:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.State - 14, // 13: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.TagsEntry - 15, // 14: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.backendRef:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef - 17, // 15: kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.TagsEntry - 1, // 16: kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.type:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.GatewayType - 2, // 17: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ip_family_mode:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.IpFamilyMode - 19, // 18: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.reachable_backends:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackends - 24, // 19: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.interval:type_name -> google.protobuf.Duration - 24, // 20: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.timeout:type_name -> google.protobuf.Duration - 25, // 21: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.unhealthy_threshold:type_name -> google.protobuf.UInt32Value - 25, // 22: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.healthy_threshold:type_name -> google.protobuf.UInt32Value - 13, // 23: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.tcp:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.Tcp - 16, // 24: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef.labels:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef.LabelsEntry - 25, // 25: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.port:type_name -> google.protobuf.UInt32Value - 20, // 26: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.labels:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.LabelsEntry - 18, // 27: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackends.refs:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef - 28, // [28:28] is the sub-list for method output_type - 28, // [28:28] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 23, // 3: kuma.mesh.v1alpha1.Dataplane.envoy:type_name -> kuma.mesh.v1alpha1.EnvoyConfiguration + 8, // 4: kuma.mesh.v1alpha1.Dataplane.Networking.gateway:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway + 6, // 5: kuma.mesh.v1alpha1.Dataplane.Networking.inbound:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound + 7, // 6: kuma.mesh.v1alpha1.Dataplane.Networking.outbound:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound + 9, // 7: kuma.mesh.v1alpha1.Dataplane.Networking.transparent_proxying:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying + 24, // 8: kuma.mesh.v1alpha1.Dataplane.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin + 21, // 9: kuma.mesh.v1alpha1.Dataplane.Probes.endpoints:type_name -> kuma.mesh.v1alpha1.Dataplane.Probes.Endpoint + 10, // 10: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.TagsEntry + 11, // 11: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.health:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.Health + 12, // 12: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.serviceProbe:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe + 0, // 13: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.state:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.State + 14, // 14: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.TagsEntry + 15, // 15: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.backendRef:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef + 17, // 16: kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.tags:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.TagsEntry + 1, // 17: kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.type:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Gateway.GatewayType + 2, // 18: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ip_family_mode:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.IpFamilyMode + 19, // 19: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.reachable_backends:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackends + 25, // 20: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.interval:type_name -> google.protobuf.Duration + 25, // 21: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.timeout:type_name -> google.protobuf.Duration + 26, // 22: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.unhealthy_threshold:type_name -> google.protobuf.UInt32Value + 26, // 23: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.healthy_threshold:type_name -> google.protobuf.UInt32Value + 13, // 24: kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.tcp:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Inbound.ServiceProbe.Tcp + 16, // 25: kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef.labels:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.Outbound.BackendRef.LabelsEntry + 26, // 26: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.port:type_name -> google.protobuf.UInt32Value + 20, // 27: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.labels:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef.LabelsEntry + 18, // 28: kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackends.refs:type_name -> kuma.mesh.v1alpha1.Dataplane.Networking.TransparentProxying.ReachableBackendRef + 29, // [29:29] is the sub-list for method output_type + 29, // [29:29] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_api_mesh_v1alpha1_dataplane_proto_init() } @@ -1717,6 +1734,7 @@ func file_api_mesh_v1alpha1_dataplane_proto_init() { } file_api_mesh_v1alpha1_metrics_proto_init() file_api_mesh_v1alpha1_envoy_admin_proto_init() + file_api_mesh_v1alpha1_envoy_proto_init() if !protoimpl.UnsafeEnabled { file_api_mesh_v1alpha1_dataplane_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Dataplane); i { diff --git a/api/mesh/v1alpha1/dataplane.proto b/api/mesh/v1alpha1/dataplane.proto index 05a8a5035607..05c0663e3e6e 100644 --- a/api/mesh/v1alpha1/dataplane.proto +++ b/api/mesh/v1alpha1/dataplane.proto @@ -7,6 +7,7 @@ option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; import "api/mesh/options.proto"; import "api/mesh/v1alpha1/metrics.proto"; import "api/mesh/v1alpha1/envoy_admin.proto"; +import "api/mesh/v1alpha1/envoy.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -369,4 +370,7 @@ message Dataplane { // Deprecated: this feature will be removed for Universal; on Kubernetes, it's // not needed anymore. Probes probes = 3; + + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + EnvoyConfiguration envoy = 4; } diff --git a/api/mesh/v1alpha1/envoy.pb.go b/api/mesh/v1alpha1/envoy.pb.go new file mode 100644 index 000000000000..c5a71a5cd965 --- /dev/null +++ b/api/mesh/v1alpha1/envoy.pb.go @@ -0,0 +1,215 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.20.0 +// source: api/mesh/v1alpha1/envoy.proto + +package v1alpha1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EnvoyConfiguration_XdsTransportProtocolVariant int32 + +const ( + // This value defines default control-plane variant. + EnvoyConfiguration_DEFAULT EnvoyConfiguration_XdsTransportProtocolVariant = 0 + // Enables incremental xDS between control-plane and the Envoy. + EnvoyConfiguration_DELTA_GRPC EnvoyConfiguration_XdsTransportProtocolVariant = 1 + // Enables state of the world xDS between control-plane and the Envoy. + EnvoyConfiguration_GRPC EnvoyConfiguration_XdsTransportProtocolVariant = 2 +) + +// Enum value maps for EnvoyConfiguration_XdsTransportProtocolVariant. +var ( + EnvoyConfiguration_XdsTransportProtocolVariant_name = map[int32]string{ + 0: "DEFAULT", + 1: "DELTA_GRPC", + 2: "GRPC", + } + EnvoyConfiguration_XdsTransportProtocolVariant_value = map[string]int32{ + "DEFAULT": 0, + "DELTA_GRPC": 1, + "GRPC": 2, + } +) + +func (x EnvoyConfiguration_XdsTransportProtocolVariant) Enum() *EnvoyConfiguration_XdsTransportProtocolVariant { + p := new(EnvoyConfiguration_XdsTransportProtocolVariant) + *p = x + return p +} + +func (x EnvoyConfiguration_XdsTransportProtocolVariant) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EnvoyConfiguration_XdsTransportProtocolVariant) Descriptor() protoreflect.EnumDescriptor { + return file_api_mesh_v1alpha1_envoy_proto_enumTypes[0].Descriptor() +} + +func (EnvoyConfiguration_XdsTransportProtocolVariant) Type() protoreflect.EnumType { + return &file_api_mesh_v1alpha1_envoy_proto_enumTypes[0] +} + +func (x EnvoyConfiguration_XdsTransportProtocolVariant) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EnvoyConfiguration_XdsTransportProtocolVariant.Descriptor instead. +func (EnvoyConfiguration_XdsTransportProtocolVariant) EnumDescriptor() ([]byte, []int) { + return file_api_mesh_v1alpha1_envoy_proto_rawDescGZIP(), []int{0, 0} +} + +type EnvoyConfiguration struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // xDSTransportProtocol provides information about protocol used for + // configuration exchange between control-plane and Envoy sidecar. + XdsTransportProtocolVariant EnvoyConfiguration_XdsTransportProtocolVariant `protobuf:"varint,1,opt,name=xds_transport_protocol_variant,json=xdsTransportProtocolVariant,proto3,enum=kuma.mesh.v1alpha1.EnvoyConfiguration_XdsTransportProtocolVariant" json:"xds_transport_protocol_variant,omitempty"` +} + +func (x *EnvoyConfiguration) Reset() { + *x = EnvoyConfiguration{} + if protoimpl.UnsafeEnabled { + mi := &file_api_mesh_v1alpha1_envoy_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EnvoyConfiguration) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvoyConfiguration) ProtoMessage() {} + +func (x *EnvoyConfiguration) ProtoReflect() protoreflect.Message { + mi := &file_api_mesh_v1alpha1_envoy_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvoyConfiguration.ProtoReflect.Descriptor instead. +func (*EnvoyConfiguration) Descriptor() ([]byte, []int) { + return file_api_mesh_v1alpha1_envoy_proto_rawDescGZIP(), []int{0} +} + +func (x *EnvoyConfiguration) GetXdsTransportProtocolVariant() EnvoyConfiguration_XdsTransportProtocolVariant { + if x != nil { + return x.XdsTransportProtocolVariant + } + return EnvoyConfiguration_DEFAULT +} + +var File_api_mesh_v1alpha1_envoy_proto protoreflect.FileDescriptor + +var file_api_mesh_v1alpha1_envoy_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x12, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x22, 0xe4, 0x01, 0x0a, 0x12, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x87, 0x01, 0x0a, 0x1e, 0x78, + 0x64, 0x73, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x42, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x58, 0x64, 0x73, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x52, 0x1b, 0x78, 0x64, 0x73, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x70, 0x6f, 0x72, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x61, 0x72, + 0x69, 0x61, 0x6e, 0x74, 0x22, 0x44, 0x0a, 0x1b, 0x58, 0x64, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x70, 0x6f, 0x72, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x61, 0x72, 0x69, + 0x61, 0x6e, 0x74, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x10, 0x00, + 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x45, 0x4c, 0x54, 0x41, 0x5f, 0x47, 0x52, 0x50, 0x43, 0x10, 0x01, + 0x12, 0x08, 0x0a, 0x04, 0x47, 0x52, 0x50, 0x43, 0x10, 0x02, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, + 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_api_mesh_v1alpha1_envoy_proto_rawDescOnce sync.Once + file_api_mesh_v1alpha1_envoy_proto_rawDescData = file_api_mesh_v1alpha1_envoy_proto_rawDesc +) + +func file_api_mesh_v1alpha1_envoy_proto_rawDescGZIP() []byte { + file_api_mesh_v1alpha1_envoy_proto_rawDescOnce.Do(func() { + file_api_mesh_v1alpha1_envoy_proto_rawDescData = protoimpl.X.CompressGZIP(file_api_mesh_v1alpha1_envoy_proto_rawDescData) + }) + return file_api_mesh_v1alpha1_envoy_proto_rawDescData +} + +var file_api_mesh_v1alpha1_envoy_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_api_mesh_v1alpha1_envoy_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_api_mesh_v1alpha1_envoy_proto_goTypes = []interface{}{ + (EnvoyConfiguration_XdsTransportProtocolVariant)(0), // 0: kuma.mesh.v1alpha1.EnvoyConfiguration.XdsTransportProtocolVariant + (*EnvoyConfiguration)(nil), // 1: kuma.mesh.v1alpha1.EnvoyConfiguration +} +var file_api_mesh_v1alpha1_envoy_proto_depIdxs = []int32{ + 0, // 0: kuma.mesh.v1alpha1.EnvoyConfiguration.xds_transport_protocol_variant:type_name -> kuma.mesh.v1alpha1.EnvoyConfiguration.XdsTransportProtocolVariant + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_api_mesh_v1alpha1_envoy_proto_init() } +func file_api_mesh_v1alpha1_envoy_proto_init() { + if File_api_mesh_v1alpha1_envoy_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_api_mesh_v1alpha1_envoy_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EnvoyConfiguration); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_api_mesh_v1alpha1_envoy_proto_rawDesc, + NumEnums: 1, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_api_mesh_v1alpha1_envoy_proto_goTypes, + DependencyIndexes: file_api_mesh_v1alpha1_envoy_proto_depIdxs, + EnumInfos: file_api_mesh_v1alpha1_envoy_proto_enumTypes, + MessageInfos: file_api_mesh_v1alpha1_envoy_proto_msgTypes, + }.Build() + File_api_mesh_v1alpha1_envoy_proto = out.File + file_api_mesh_v1alpha1_envoy_proto_rawDesc = nil + file_api_mesh_v1alpha1_envoy_proto_goTypes = nil + file_api_mesh_v1alpha1_envoy_proto_depIdxs = nil +} diff --git a/api/mesh/v1alpha1/envoy.proto b/api/mesh/v1alpha1/envoy.proto new file mode 100644 index 000000000000..829cd125de54 --- /dev/null +++ b/api/mesh/v1alpha1/envoy.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package kuma.mesh.v1alpha1; + +option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; + +message EnvoyConfiguration { + enum XdsTransportProtocolVariant { + // This value defines default control-plane variant. + DEFAULT = 0; + // Enables incremental xDS between control-plane and the Envoy. + DELTA_GRPC = 1; + // Enables state of the world xDS between control-plane and the Envoy. + GRPC = 2; + } + // xDSTransportProtocol provides information about protocol used for + // configuration exchange between control-plane and Envoy sidecar. + XdsTransportProtocolVariant xds_transport_protocol_variant = 1; +} diff --git a/api/mesh/v1alpha1/zone_ingress.pb.go b/api/mesh/v1alpha1/zone_ingress.pb.go index 027561423e55..07be1fd05efa 100644 --- a/api/mesh/v1alpha1/zone_ingress.pb.go +++ b/api/mesh/v1alpha1/zone_ingress.pb.go @@ -38,6 +38,8 @@ type ZoneIngress struct { // AvailableService contains tags that represent unique subset of // endpoints AvailableServices []*ZoneIngress_AvailableService `protobuf:"bytes,3,rep,name=availableServices,proto3" json:"availableServices,omitempty"` + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + Envoy *EnvoyConfiguration `protobuf:"bytes,4,opt,name=envoy,proto3" json:"envoy,omitempty"` } func (x *ZoneIngress) Reset() { @@ -93,6 +95,13 @@ func (x *ZoneIngress) GetAvailableServices() []*ZoneIngress_AvailableService { return nil } +func (x *ZoneIngress) GetEnvoy() *EnvoyConfiguration { + if x != nil { + return x.Envoy + } + return nil +} + type ZoneIngress_Networking struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -262,68 +271,74 @@ var file_api_mesh_v1alpha1_zone_ingress_proto_rawDesc = []byte{ 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x16, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x1a, 0x23, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, - 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf6, 0x06, 0x0a, 0x0b, 0x5a, 0x6f, 0x6e, 0x65, - 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x4a, 0x0a, 0x0a, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x2a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x2e, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, - 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x52, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x1a, 0xc6, 0x01, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, - 0x12, 0x2c, 0x0a, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, - 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, - 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, - 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, - 0x72, 0x74, 0x12, 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, - 0x50, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x61, 0x64, 0x76, 0x65, - 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x61, 0x64, - 0x6d, 0x69, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6b, 0x75, 0x6d, 0x61, - 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, - 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x52, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, - 0x1a, 0xf7, 0x01, 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x12, 0x28, 0x0a, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x1a, 0x37, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x3a, 0xe3, 0x01, 0xaa, 0x8c, 0x89, - 0xa6, 0x01, 0x15, 0x0a, 0x13, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0d, 0x12, 0x0b, - 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, - 0x02, 0x18, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x06, 0x22, 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, - 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x08, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, - 0x02, 0x10, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x10, 0x3a, 0x0e, 0x0a, 0x0c, 0x7a, 0x6f, 0x6e, - 0x65, 0x2d, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x12, 0x3a, - 0x10, 0x12, 0x0e, 0x7a, 0x6f, 0x6e, 0x65, 0x2d, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x65, - 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x03, 0x90, - 0x01, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x48, 0x8a, 0x01, 0x45, 0x64, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x5a, 0x6f, 0x6e, 0x65, 0x20, 0x6e, 0x61, 0x6d, - 0x65, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, - 0x65, 0x63, 0x2e, 0x7a, 0x6f, 0x6e, 0x65, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x7a, - 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, - 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, - 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x6f, 0x1a, 0x1d, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x23, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb4, 0x07, 0x0a, 0x0b, 0x5a, 0x6f, 0x6e, 0x65, 0x49, + 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x4a, 0x0a, 0x0a, 0x6e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, + 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x5e, 0x0a, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, + 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x30, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, + 0x73, 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x11, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x05, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, + 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x65, + 0x6e, 0x76, 0x6f, 0x79, 0x1a, 0xc6, 0x01, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x69, 0x6e, 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x2c, 0x0a, + 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, + 0x69, 0x73, 0x65, 0x64, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, + 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, + 0x26, 0x0a, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x61, 0x64, 0x76, 0x65, 0x72, 0x74, 0x69, + 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, + 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, + 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x52, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x1a, 0xf7, 0x01, + 0x0a, 0x10, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x3a, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x2e, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x09, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6d, 0x65, 0x73, 0x68, 0x12, 0x28, 0x0a, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x65, + 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x1a, 0x37, + 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x3a, 0xe3, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x15, + 0x0a, 0x13, 0x5a, 0x6f, 0x6e, 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0d, 0x12, 0x0b, 0x5a, 0x6f, 0x6e, + 0x65, 0x49, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x18, 0x01, + 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x06, 0x22, 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, 0x8c, 0x89, 0xa6, + 0x01, 0x04, 0x52, 0x02, 0x08, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x10, 0x01, + 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x10, 0x3a, 0x0e, 0x0a, 0x0c, 0x7a, 0x6f, 0x6e, 0x65, 0x2d, 0x69, + 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x12, 0x3a, 0x10, 0x12, 0x0e, + 0x7a, 0x6f, 0x6e, 0x65, 0x2d, 0x69, 0x6e, 0x67, 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0xaa, 0x8c, + 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x03, 0x90, 0x01, 0x01, 0xaa, + 0x8c, 0x89, 0xa6, 0x01, 0x48, 0x8a, 0x01, 0x45, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x5a, 0x6f, 0x6e, 0x65, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x2c, + 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, + 0x7a, 0x6f, 0x6e, 0x65, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, 0x22, 0x7a, 0x6f, 0x6e, 0x65, + 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x2a, 0x5a, + 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, + 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, + 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -344,18 +359,20 @@ var file_api_mesh_v1alpha1_zone_ingress_proto_goTypes = []interface{}{ (*ZoneIngress_Networking)(nil), // 1: kuma.mesh.v1alpha1.ZoneIngress.Networking (*ZoneIngress_AvailableService)(nil), // 2: kuma.mesh.v1alpha1.ZoneIngress.AvailableService nil, // 3: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry - (*EnvoyAdmin)(nil), // 4: kuma.mesh.v1alpha1.EnvoyAdmin + (*EnvoyConfiguration)(nil), // 4: kuma.mesh.v1alpha1.EnvoyConfiguration + (*EnvoyAdmin)(nil), // 5: kuma.mesh.v1alpha1.EnvoyAdmin } var file_api_mesh_v1alpha1_zone_ingress_proto_depIdxs = []int32{ 1, // 0: kuma.mesh.v1alpha1.ZoneIngress.networking:type_name -> kuma.mesh.v1alpha1.ZoneIngress.Networking 2, // 1: kuma.mesh.v1alpha1.ZoneIngress.availableServices:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService - 4, // 2: kuma.mesh.v1alpha1.ZoneIngress.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin - 3, // 3: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.tags:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 4, // 2: kuma.mesh.v1alpha1.ZoneIngress.envoy:type_name -> kuma.mesh.v1alpha1.EnvoyConfiguration + 5, // 3: kuma.mesh.v1alpha1.ZoneIngress.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin + 3, // 4: kuma.mesh.v1alpha1.ZoneIngress.AvailableService.tags:type_name -> kuma.mesh.v1alpha1.ZoneIngress.AvailableService.TagsEntry + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_api_mesh_v1alpha1_zone_ingress_proto_init() } @@ -363,6 +380,7 @@ func file_api_mesh_v1alpha1_zone_ingress_proto_init() { if File_api_mesh_v1alpha1_zone_ingress_proto != nil { return } + file_api_mesh_v1alpha1_envoy_proto_init() file_api_mesh_v1alpha1_envoy_admin_proto_init() if !protoimpl.UnsafeEnabled { file_api_mesh_v1alpha1_zone_ingress_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { diff --git a/api/mesh/v1alpha1/zone_ingress.proto b/api/mesh/v1alpha1/zone_ingress.proto index 54752cdb99f7..a80c4cf5cd9a 100644 --- a/api/mesh/v1alpha1/zone_ingress.proto +++ b/api/mesh/v1alpha1/zone_ingress.proto @@ -5,6 +5,7 @@ package kuma.mesh.v1alpha1; option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; import "api/mesh/options.proto"; +import "api/mesh/v1alpha1/envoy.proto"; import "api/mesh/v1alpha1/envoy_admin.proto"; // ZoneIngress allows us to configure dataplane in the Ingress mode. In this @@ -68,4 +69,7 @@ message ZoneIngress { // AvailableService contains tags that represent unique subset of // endpoints repeated AvailableService availableServices = 3; + + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + EnvoyConfiguration envoy = 4; } diff --git a/api/mesh/v1alpha1/zoneegress.pb.go b/api/mesh/v1alpha1/zoneegress.pb.go index 20e9ade5e8ca..04279ddb987d 100644 --- a/api/mesh/v1alpha1/zoneegress.pb.go +++ b/api/mesh/v1alpha1/zoneegress.pb.go @@ -32,6 +32,8 @@ type ZoneEgress struct { Zone string `protobuf:"bytes,1,opt,name=zone,proto3" json:"zone,omitempty"` // Networking defines the address and port of the Egress to listen on. Networking *ZoneEgress_Networking `protobuf:"bytes,2,opt,name=networking,proto3" json:"networking,omitempty"` + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + Envoy *EnvoyConfiguration `protobuf:"bytes,3,opt,name=envoy,proto3" json:"envoy,omitempty"` } func (x *ZoneEgress) Reset() { @@ -80,6 +82,13 @@ func (x *ZoneEgress) GetNetworking() *ZoneEgress_Networking { return nil } +func (x *ZoneEgress) GetEnvoy() *EnvoyConfiguration { + if x != nil { + return x.Envoy + } + return nil +} + type ZoneEgress_Networking struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -154,40 +163,46 @@ var file_api_mesh_v1alpha1_zoneegress_proto_rawDesc = []byte{ 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x16, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x1a, 0x23, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbd, 0x03, 0x0a, 0x0a, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x49, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6b, - 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x69, 0x6e, 0x67, 0x1a, 0x70, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, - 0x67, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, - 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, - 0x34, 0x0a, 0x05, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, - 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x52, 0x05, - 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x3a, 0xdd, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x14, 0x0a, 0x12, - 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0c, 0x12, 0x0a, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, - 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x18, 0x01, 0xaa, 0x8c, 0x89, 0xa6, - 0x01, 0x06, 0x22, 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, - 0x08, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x10, 0x00, 0xaa, 0x8c, 0x89, 0xa6, - 0x01, 0x0e, 0x3a, 0x0c, 0x0a, 0x0a, 0x7a, 0x6f, 0x6e, 0x65, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, - 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x10, 0x3a, 0x0e, 0x12, 0x0c, 0x7a, 0x6f, 0x6e, 0x65, 0x65, 0x67, - 0x72, 0x65, 0x73, 0x73, 0x65, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, 0xaa, 0x8c, - 0x89, 0xa6, 0x01, 0x03, 0x90, 0x01, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x48, 0x8a, 0x01, 0x45, - 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x5a, 0x6f, 0x6e, - 0x65, 0x20, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, - 0x3d, 0x60, 0x2e, 0x73, 0x70, 0x65, 0x63, 0x2e, 0x7a, 0x6f, 0x6e, 0x65, 0x60, 0x2c, 0x6e, 0x61, - 0x6d, 0x65, 0x3d, 0x22, 0x7a, 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, - 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x1a, 0x1d, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x23, 0x61, 0x70, 0x69, 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2f, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x5f, 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xfb, 0x03, 0x0a, 0x0a, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x49, 0x0a, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x6b, 0x75, + 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x52, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, + 0x6e, 0x67, 0x12, 0x3c, 0x0a, 0x05, 0x65, 0x6e, 0x76, 0x6f, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x26, 0x2e, 0x6b, 0x75, 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x65, 0x6e, 0x76, 0x6f, 0x79, + 0x1a, 0x70, 0x0a, 0x0a, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x12, 0x18, + 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x34, 0x0a, 0x05, + 0x61, 0x64, 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6b, 0x75, + 0x6d, 0x61, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x2e, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x52, 0x05, 0x61, 0x64, 0x6d, + 0x69, 0x6e, 0x3a, 0xdd, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x14, 0x0a, 0x12, 0x5a, 0x6f, 0x6e, + 0x65, 0x45, 0x67, 0x72, 0x65, 0x73, 0x73, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xaa, + 0x8c, 0x89, 0xa6, 0x01, 0x0c, 0x12, 0x0a, 0x5a, 0x6f, 0x6e, 0x65, 0x45, 0x67, 0x72, 0x65, 0x73, + 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x18, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x06, 0x22, + 0x04, 0x6d, 0x65, 0x73, 0x68, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x08, 0x01, 0xaa, + 0x8c, 0x89, 0xa6, 0x01, 0x04, 0x52, 0x02, 0x10, 0x00, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x0e, 0x3a, + 0x0c, 0x0a, 0x0a, 0x7a, 0x6f, 0x6e, 0x65, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, 0xaa, 0x8c, 0x89, + 0xa6, 0x01, 0x10, 0x3a, 0x0e, 0x12, 0x0c, 0x7a, 0x6f, 0x6e, 0x65, 0x65, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x73, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x02, 0x58, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, + 0x03, 0x90, 0x01, 0x01, 0xaa, 0x8c, 0x89, 0xa6, 0x01, 0x48, 0x8a, 0x01, 0x45, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x3d, 0x22, 0x5a, 0x6f, 0x6e, 0x65, 0x20, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x2c, 0x4a, 0x53, 0x4f, 0x4e, 0x50, 0x61, 0x74, 0x68, 0x3d, 0x60, 0x2e, + 0x73, 0x70, 0x65, 0x63, 0x2e, 0x7a, 0x6f, 0x6e, 0x65, 0x60, 0x2c, 0x6e, 0x61, 0x6d, 0x65, 0x3d, + 0x22, 0x7a, 0x6f, 0x6e, 0x65, 0x22, 0x2c, 0x74, 0x79, 0x70, 0x65, 0x3d, 0x73, 0x74, 0x72, 0x69, + 0x6e, 0x67, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x68, 0x71, 0x2f, 0x6b, 0x75, 0x6d, 0x61, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -206,16 +221,18 @@ var file_api_mesh_v1alpha1_zoneegress_proto_msgTypes = make([]protoimpl.MessageI var file_api_mesh_v1alpha1_zoneegress_proto_goTypes = []interface{}{ (*ZoneEgress)(nil), // 0: kuma.mesh.v1alpha1.ZoneEgress (*ZoneEgress_Networking)(nil), // 1: kuma.mesh.v1alpha1.ZoneEgress.Networking - (*EnvoyAdmin)(nil), // 2: kuma.mesh.v1alpha1.EnvoyAdmin + (*EnvoyConfiguration)(nil), // 2: kuma.mesh.v1alpha1.EnvoyConfiguration + (*EnvoyAdmin)(nil), // 3: kuma.mesh.v1alpha1.EnvoyAdmin } var file_api_mesh_v1alpha1_zoneegress_proto_depIdxs = []int32{ 1, // 0: kuma.mesh.v1alpha1.ZoneEgress.networking:type_name -> kuma.mesh.v1alpha1.ZoneEgress.Networking - 2, // 1: kuma.mesh.v1alpha1.ZoneEgress.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin - 2, // [2:2] is the sub-list for method output_type - 2, // [2:2] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 2, // 1: kuma.mesh.v1alpha1.ZoneEgress.envoy:type_name -> kuma.mesh.v1alpha1.EnvoyConfiguration + 3, // 2: kuma.mesh.v1alpha1.ZoneEgress.Networking.admin:type_name -> kuma.mesh.v1alpha1.EnvoyAdmin + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_api_mesh_v1alpha1_zoneegress_proto_init() } @@ -223,6 +240,7 @@ func file_api_mesh_v1alpha1_zoneegress_proto_init() { if File_api_mesh_v1alpha1_zoneegress_proto != nil { return } + file_api_mesh_v1alpha1_envoy_proto_init() file_api_mesh_v1alpha1_envoy_admin_proto_init() if !protoimpl.UnsafeEnabled { file_api_mesh_v1alpha1_zoneegress_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { diff --git a/api/mesh/v1alpha1/zoneegress.proto b/api/mesh/v1alpha1/zoneegress.proto index e61b57d877f9..fbd04d25e9b9 100644 --- a/api/mesh/v1alpha1/zoneegress.proto +++ b/api/mesh/v1alpha1/zoneegress.proto @@ -5,6 +5,7 @@ package kuma.mesh.v1alpha1; option go_package = "github.com/kumahq/kuma/api/mesh/v1alpha1"; import "api/mesh/options.proto"; +import "api/mesh/v1alpha1/envoy.proto"; import "api/mesh/v1alpha1/envoy_admin.proto"; // ZoneEgress allows us to configure dataplane in the Egress mode. @@ -42,4 +43,7 @@ message ZoneEgress { // Networking defines the address and port of the Egress to listen on. Networking networking = 2; + + // EnvoyConfiguration provides additional configuration for the Envoy sidecar. + EnvoyConfiguration envoy = 3; } diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go index e837020e43a8..cae5221c7112 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap.go @@ -191,14 +191,6 @@ func (b *remoteBootstrap) requestForBootstrap(ctx context.Context, client *http. }, SystemCaPath: params.SystemCaPath, } - switch cfg.DataplaneRuntime.XdsConfigMode { - case "delta": - request.XdsConfigMode = types.DELTA - case "sotw": - request.XdsConfigMode = types.SOTW - default: - request.XdsConfigMode = types.NOT_DEFINED - } jsonBytes, err := json.MarshalIndent(request, "", " ") if err != nil { return nil, errors.Wrap(err, "could not marshal request to json") diff --git a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go index 08f9e9ff5360..1a0882e80a74 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go +++ b/app/kuma-dp/pkg/dataplane/envoy/remote_bootstrap_test.go @@ -149,7 +149,6 @@ var _ = Describe("Remote Bootstrap", func() { cfg.Dataplane.Mesh = "demo" cfg.Dataplane.Name = "sample" cfg.DataplaneRuntime.Token = "token" - cfg.DataplaneRuntime.XdsConfigMode = "delta" return testCase{ config: cfg, diff --git a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json index 15328d0acc5e..473d5360187d 100644 --- a/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json +++ b/app/kuma-dp/pkg/dataplane/envoy/testdata/bootstrap-request-3.golden.json @@ -29,6 +29,5 @@ "certPath": "", "keyPath": "" }, - "systemCaPath": "", - "xdsConfigMode": 2 + "systemCaPath": "" } diff --git a/docs/generated/raw/protos/Dataplane.json b/docs/generated/raw/protos/Dataplane.json index 38967b2d4fd2..ddc91151d845 100644 --- a/docs/generated/raw/protos/Dataplane.json +++ b/docs/generated/raw/protos/Dataplane.json @@ -18,6 +18,11 @@ "$ref": "#/definitions/kuma.mesh.v1alpha1.Dataplane.Probes", "additionalProperties": true, "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information. Deprecated: this feature will be removed for Universal; on Kubernetes, it's not needed anymore." + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, @@ -420,6 +425,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.MetricsBackend": { "properties": { "name": { diff --git a/docs/generated/raw/protos/DataplaneOverview.json b/docs/generated/raw/protos/DataplaneOverview.json index 48d6b0e65093..2ec97724cbc2 100644 --- a/docs/generated/raw/protos/DataplaneOverview.json +++ b/docs/generated/raw/protos/DataplaneOverview.json @@ -34,6 +34,11 @@ "$ref": "#/definitions/kuma.mesh.v1alpha1.Dataplane.Probes", "additionalProperties": true, "description": "Probes describe a list of endpoints that will be exposed without mTLS. This is useful to expose the health endpoints of the application so the orchestration system (e.g. Kubernetes) can still health check the application. See https://kuma.io/docs/latest/policies/service-health-probes/#virtual-probes for more information. Deprecated: this feature will be removed for Universal; on Kubernetes, it's not needed anymore." + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, @@ -589,6 +594,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.EnvoyVersion": { "properties": { "version": { diff --git a/docs/generated/raw/protos/EnvoyConfiguration.json b/docs/generated/raw/protos/EnvoyConfiguration.json new file mode 100644 index 000000000000..9d788233751a --- /dev/null +++ b/docs/generated/raw/protos/EnvoyConfiguration.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "#/definitions/EnvoyConfiguration", + "definitions": { + "EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + } + } +} \ No newline at end of file diff --git a/docs/generated/raw/protos/ZoneEgress.json b/docs/generated/raw/protos/ZoneEgress.json index b154ca74f1c0..c87af4b93f0e 100644 --- a/docs/generated/raw/protos/ZoneEgress.json +++ b/docs/generated/raw/protos/ZoneEgress.json @@ -12,6 +12,11 @@ "$ref": "#/definitions/kuma.mesh.v1alpha1.ZoneEgress.Networking", "additionalProperties": true, "description": "Networking defines the address and port of the Egress to listen on." + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, @@ -30,6 +35,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.ZoneEgress.Networking": { "properties": { "address": { diff --git a/docs/generated/raw/protos/ZoneEgressOverview.json b/docs/generated/raw/protos/ZoneEgressOverview.json index 7e2fe59831a0..27c87d004fc6 100644 --- a/docs/generated/raw/protos/ZoneEgressOverview.json +++ b/docs/generated/raw/protos/ZoneEgressOverview.json @@ -129,6 +129,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.EnvoyVersion": { "properties": { "version": { @@ -212,6 +238,11 @@ "$ref": "#/definitions/kuma.mesh.v1alpha1.ZoneEgress.Networking", "additionalProperties": true, "description": "Networking defines the address and port of the Egress to listen on." + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, diff --git a/docs/generated/raw/protos/ZoneIngress.json b/docs/generated/raw/protos/ZoneIngress.json index 0d613e59b360..cfee769546f2 100644 --- a/docs/generated/raw/protos/ZoneIngress.json +++ b/docs/generated/raw/protos/ZoneIngress.json @@ -19,6 +19,11 @@ }, "type": "array", "description": "AvailableService contains tags that represent unique subset of endpoints" + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, @@ -37,6 +42,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.ZoneIngress.AvailableService": { "properties": { "tags": { diff --git a/docs/generated/raw/protos/ZoneIngressOverview.json b/docs/generated/raw/protos/ZoneIngressOverview.json index 5cf6ef549cfb..a5b2a0bffd25 100644 --- a/docs/generated/raw/protos/ZoneIngressOverview.json +++ b/docs/generated/raw/protos/ZoneIngressOverview.json @@ -129,6 +129,32 @@ "type": "object", "title": "Envoy Admin" }, + "kuma.mesh.v1alpha1.EnvoyConfiguration": { + "properties": { + "xds_transport_protocol_variant": { + "enum": [ + "DEFAULT", + 0, + "DELTA_GRPC", + 1, + "GRPC", + 2 + ], + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ], + "title": "Xds Transport Protocol Variant" + } + }, + "additionalProperties": true, + "type": "object", + "title": "Envoy Configuration" + }, "kuma.mesh.v1alpha1.EnvoyVersion": { "properties": { "version": { @@ -219,6 +245,11 @@ }, "type": "array", "description": "AvailableService contains tags that represent unique subset of endpoints" + }, + "envoy": { + "$ref": "#/definitions/kuma.mesh.v1alpha1.EnvoyConfiguration", + "additionalProperties": true, + "description": "EnvoyConfiguration provides additional configuration for the Envoy sidecar." } }, "additionalProperties": true, diff --git a/pkg/config/app/kuma-cp/config.go b/pkg/config/app/kuma-cp/config.go index 4529711c7209..d458f74232f9 100644 --- a/pkg/config/app/kuma-cp/config.go +++ b/pkg/config/app/kuma-cp/config.go @@ -465,7 +465,7 @@ type ExperimentalConfig struct { // Do not enable on production. SkipPersistedVIPs bool `json:"skipPersistedVIPs" envconfig:"KUMA_EXPERIMENTAL_SKIP_PERSISTED_VIPS"` // If true uses Delta xDS to deliver changes to sidecars. - UseDeltaXds bool `json:"useDeltaXds" envconfig:"KUMA_EXPERIMENTAL_USE_DELTA_XDS"` + DeltaXds bool `json:"deltaXds" envconfig:"KUMA_EXPERIMENTAL_DELTA_XDS"` } type ExperimentalKDSEventBasedWatchdog struct { diff --git a/pkg/config/app/kuma-dp/config.go b/pkg/config/app/kuma-dp/config.go index bd0e5b360f9a..05e94feadebf 100644 --- a/pkg/config/app/kuma-dp/config.go +++ b/pkg/config/app/kuma-dp/config.go @@ -217,10 +217,6 @@ type DataplaneRuntime struct { DynamicConfiguration DynamicConfiguration `json:"dynamicConfiguration" envconfig:"kuma_dataplane_runtime_dynamic_configuration"` // SystemCaPath defines path of system provided Ca SystemCaPath string `json:"systemCaPath,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_system_ca_path"` - // XdsConfigMode defines xDS communication type between Envoy and control-plane. - // Available values are: sotw, delta - // By default it takes configuration provided by the control-plane. - XdsConfigMode string `json:"xdsConfigMode,omitempty" envconfig:"kuma_dataplane_runtime_dynamic_xds_config_mode"` } type Metrics struct { @@ -339,13 +335,6 @@ func (d *DataplaneRuntime) Validate() error { if d.BinaryPath == "" { errs = multierr.Append(errs, errors.Errorf(".BinaryPath must be non-empty")) } - if d.XdsConfigMode != "" { - switch d.XdsConfigMode { - case "delta", "sotw": - default: - errs = multierr.Append(errs, errors.Errorf(".XdsConfigMode can be one of: delta, sotw")) - } - } return errs } diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt index fedc368f1e63..5ca12650bb24 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.golden.txt @@ -1 +1 @@ -parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty; .XdsConfigMode can be one of: delta, sotw +parsing configuration from file 'testdata/invalid-config.input.yaml' failed: configuration validation failed: .ControlPlane is not valid: .Retry is not valid: .Backoff must be a positive duration; .Dataplane is not valid: .ProxyType is not valid: not-a-proxy is not a valid proxy type; .Mesh must be non-empty; .Name must be non-empty; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty diff --git a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml index daf21d15719b..7467f0431afa 100644 --- a/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml @@ -12,4 +12,3 @@ dataplane: proxyType: not-a-proxy dataplaneRuntime: binaryPath: - xdsConfigMode: a diff --git a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml index e79cb0c531d2..5046ac0d7784 100644 --- a/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml +++ b/pkg/config/app/kuma-dp/testdata/valid-config.input.yaml @@ -12,4 +12,3 @@ dataplaneRuntime: binaryPath: envoy.sh configDir: /var/run/envoy envoyLogLevel: trace - xdsConfigMode: delta diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 26adac35258f..f76e6776d429 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -372,7 +372,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Experimental.SidecarContainers).To(BeTrue()) Expect(cfg.Experimental.SkipPersistedVIPs).To(BeTrue()) Expect(cfg.Experimental.GenerateMeshServices).To(BeTrue()) - Expect(cfg.Experimental.UseDeltaXds).To(BeTrue()) + Expect(cfg.Experimental.DeltaXds).To(BeTrue()) Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1)) Expect(cfg.EventBus.BufferSize).To(Equal(uint(30))) @@ -760,7 +760,7 @@ experimental: sidecarContainers: true generateMeshServices: true skipPersistedVIPs: true - useDeltaXds: true + deltaXds: true proxy: gateway: globalDownstreamMaxConnections: 1 @@ -1067,7 +1067,7 @@ meshService: "KUMA_EXPERIMENTAL_SIDECAR_CONTAINERS": "true", "KUMA_EXPERIMENTAL_GENERATE_MESH_SERVICES": "true", "KUMA_EXPERIMENTAL_SKIP_PERSISTED_VIPS": "true", - "KUMA_EXPERIMENTAL_USE_DELTA_XDS": "true", + "KUMA_EXPERIMENTAL_DELTA_XDS": "true", "KUMA_PROXY_GATEWAY_GLOBAL_DOWNSTREAM_MAX_CONNECTIONS": "1", "KUMA_TRACING_OPENTELEMETRY_ENDPOINT": "otel-collector:4317", "KUMA_TRACING_OPENTELEMETRY_ENABLED": "true", diff --git a/pkg/plugins/runtime/k8s/controllers/egress_converter.go b/pkg/plugins/runtime/k8s/controllers/egress_converter.go index f23bee3cbd5a..36638ac9e817 100644 --- a/pkg/plugins/runtime/k8s/controllers/egress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/egress_converter.go @@ -42,6 +42,7 @@ func (p *PodConverter) EgressFor( if exist { zoneEgress.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } + zoneEgress.Envoy = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) return nil } diff --git a/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go b/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go new file mode 100644 index 000000000000..6d169196ae11 --- /dev/null +++ b/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go @@ -0,0 +1,25 @@ +package controllers + +import ( + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" + "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" +) + +func GetEnvoyConfiguration(deltaXds bool, annotations metadata.Annotations) *mesh_proto.EnvoyConfiguration { + envoyConfig := &mesh_proto.EnvoyConfiguration{ + XdsTransportProtocolVariant: mesh_proto.EnvoyConfiguration_GRPC, + } + if deltaXds { + envoyConfig.XdsTransportProtocolVariant = mesh_proto.EnvoyConfiguration_DELTA_GRPC + } + xdsTransportProtocolVariant, exist := annotations.GetString(metadata.KumaXdsTransportProtocolVariant) + if exist { + switch xdsTransportProtocolVariant { + case "DELTA_GRPC": + envoyConfig.XdsTransportProtocolVariant = mesh_proto.EnvoyConfiguration_DELTA_GRPC + case "GRPC": + envoyConfig.XdsTransportProtocolVariant = mesh_proto.EnvoyConfiguration_GRPC + } + } + return envoyConfig +} diff --git a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go index 7e250b01013e..e4db7029cb7f 100644 --- a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go @@ -66,6 +66,7 @@ func (p *PodConverter) IngressFor( if exist { zoneIngress.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } + zoneIngress.Envoy = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) return nil } diff --git a/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go b/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go index 98569c025268..ff60f99e18fd 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_controller_test.go @@ -649,6 +649,8 @@ var _ = Describe("PodReconciler", func() { uid: pod-with-kuma-sidecar-and-ip-demo resourceVersion: "1" spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: @@ -742,6 +744,8 @@ var _ = Describe("PodReconciler", func() { uid: "pod-with-kuma-sidecar-and-ip-demo" resourceVersion: "2" spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: @@ -808,6 +812,9 @@ var _ = Describe("PodReconciler", func() { }, }, Spec: mesh_k8s.ToSpec(&mesh_proto.Dataplane{ + Envoy: &mesh_proto.EnvoyConfiguration{ + XdsTransportProtocolVariant: mesh_proto.EnvoyConfiguration_GRPC, + }, Networking: &mesh_proto.Dataplane_Networking{ Address: "192.168.0.1", Admin: &mesh_proto.EnvoyAdmin{ @@ -892,6 +899,8 @@ var _ = Describe("PodReconciler", func() { uid: pod-with-custom-admin-port-demo resourceVersion: "1" spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 admin: @@ -1008,6 +1017,8 @@ var _ = Describe("PodReconciler", func() { uid: pod-with-custom-admin-port-demo resourceVersion: "2" spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 admin: diff --git a/pkg/plugins/runtime/k8s/controllers/pod_converter.go b/pkg/plugins/runtime/k8s/controllers/pod_converter.go index cc5218f87a6e..5aba2d53d319 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_converter.go @@ -38,6 +38,7 @@ type PodConverter struct { SystemNamespace string Mode config_core.CpMode KubeOutboundsAsVIPs bool + DeltaXds bool } func (p *PodConverter) PodToDataplane( @@ -300,7 +301,7 @@ func (p *PodConverter) dataplaneFor( if exist { dataplane.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } - + dataplane.Envoy = GetEnvoyConfiguration(p.DeltaXds, annotations) return dataplane, nil } diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/01.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/01.dataplane.yaml index 61edaf95253c..21bf096e1f8b 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/01.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/01.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/02.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/02.dataplane.yaml index 0bade8fc8d5b..29d19f4e96e9 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/02.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/02.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/03.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/03.dataplane.yaml index 6a4b284ac2cb..d42be6335320 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/03.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/03.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/04.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/04.dataplane.yaml index 802ef6d31c31..00f1892d7668 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/04.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/04.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/05.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/05.dataplane.yaml index 87baef58141b..4d82d9166b14 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/05.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/05.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/06.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/06.dataplane.yaml index 0ed9328673d4..7c1fffcbf911 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/06.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/06.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/07.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/07.dataplane.yaml index 1ae771bf5fd0..aba0b4e103e0 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/07.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/07.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC metrics: conf: path: /non-standard-path diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/08.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/08.dataplane.yaml index d86110709779..ab9abcd73177 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/08.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/08.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/09.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/09.dataplane.yaml index 218c38939473..ce56b479699e 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/09.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/09.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 10.244.0.8 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/10.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/10.dataplane.yaml index f813c4698b9e..d80d334fe3da 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/10.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/10.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/11.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/11.dataplane.yaml index ba6e9eb98cf3..4b8ea4952904 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/11.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/11.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/12.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/12.dataplane.yaml index cd7f98674430..e76099ee3fde 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/12.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/12.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/13.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/13.dataplane.yaml index e2c515b74ae7..ab910aa28fa1 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/13.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/13.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/14.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/14.dataplane.yaml index 251cfc272e60..84c1aebd2bdc 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/14.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/14.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/15.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/15.dataplane.yaml index cee869f1866a..370ac520da45 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/15.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/15.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: fd00::1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/16.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/16.dataplane.yaml index 36ede3a948fc..adb95d2d3a69 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/16.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/16.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 10.244.0.8 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/17.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/17.dataplane.yaml index 37c97372226e..9cccaf4d810c 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/17.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/17.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/18.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/18.dataplane.yaml index 43ab69dac20b..e6f3536dea35 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/18.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/18.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/19.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/19.dataplane.yaml index c496f9cca52f..d54aa8574092 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/19.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/19.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/20.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/20.dataplane.yaml index 380e797dd9d3..f188e895c1fc 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/20.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/20.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/21.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/21.dataplane.yaml index 6a4b284ac2cb..d42be6335320 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/21.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/21.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/22.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/22.dataplane.yaml index 6a4b284ac2cb..d42be6335320 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/22.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/22.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 gateway: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/23.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/23.dataplane.yaml index 4d292459bf6c..dd5ccb4194c1 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/23.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/23.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/24.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/24.dataplane.yaml index 072d4a856d62..573bfba9cc77 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/24.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/24.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/25.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/25.dataplane.yaml index 072d4a856d62..573bfba9cc77 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/25.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/25.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/26.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/26.dataplane.yaml index fb64a8762616..2c44fdecca9a 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/26.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/26.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/27.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/27.dataplane.yaml index bedb89fb04c3..40f0a32c2ac6 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/27.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/27.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/28.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/28.dataplane.yaml index 483a25871cf0..33caa9b9af13 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/28.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/28.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/29.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/29.dataplane.yaml index 0b2fc5ab593c..258392c83e51 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/29.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/29.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/30.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/30.dataplane.yaml index f12053f72647..a3c76228c467 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/30.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/30.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: DELTA_GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/30.pod.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/30.pod.yaml index 3d6919a3e1c4..ad21b10ab6d8 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/30.pod.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/30.pod.yaml @@ -8,6 +8,7 @@ metadata: kuma.io/virtual-probes: enabled kuma.io/virtual-probes-port: 19000 kuma.io/application-probe-proxy-port: "8662" + kuma.io/xds-transport-protocol-variant: "DELTA_GRPC" spec: containers: - readinessProbe: diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/egress/01.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/egress/01.dataplane.yaml index e8151f62f939..bad0c056f2fb 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/egress/01.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/egress/01.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/egress/02.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/egress/02.dataplane.yaml index e3e12beb5484..a71b62c34901 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/egress/02.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/egress/02.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10002 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/egress/03.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/egress/03.dataplane.yaml index e3e12beb5484..a71b62c34901 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/egress/03.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/egress/03.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10002 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/egress/04.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/egress/04.dataplane.yaml index e3e12beb5484..a71b62c34901 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/egress/04.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/egress/04.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10002 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/egress/05.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/egress/05.dataplane.yaml index e3e12beb5484..a71b62c34901 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/egress/05.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/egress/05.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10002 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml index d858a920d036..98baef2455b7 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/01.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: kuma-ingress.com diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml index 86fa5fc44aa8..94144ca4d789 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/02.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: 192.168.100.1 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml index 1b076fd466c8..a9c9bf5b9a2c 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/03.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 port: 10001 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml index 87e89877c91e..568b37cf3de3 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/04.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: 34.72.129.131 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml index 75c533fe8446..f49c1a6392c1 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/05.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: 10.128.15.193 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml index 429d8c83ce6a..9a766d62b2d6 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/06.dataplane.yaml @@ -1,6 +1,8 @@ metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: custom-address.com diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/ingress-exists.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/ingress-exists.dataplane.yaml index 74ebdeff6120..0edc16fe4277 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/ingress/ingress-exists.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/ingress/ingress-exists.dataplane.yaml @@ -7,6 +7,8 @@ spec: tags: kuma.io/protocol: http kuma.io/service: service-1-zone-2 + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 advertisedAddress: 192.168.100.1 diff --git a/pkg/plugins/runtime/k8s/controllers/testdata/mismatch-ports.dataplane.yaml b/pkg/plugins/runtime/k8s/controllers/testdata/mismatch-ports.dataplane.yaml index e2c515b74ae7..ab910aa28fa1 100644 --- a/pkg/plugins/runtime/k8s/controllers/testdata/mismatch-ports.dataplane.yaml +++ b/pkg/plugins/runtime/k8s/controllers/testdata/mismatch-ports.dataplane.yaml @@ -2,6 +2,8 @@ mesh: default metadata: creationTimestamp: null spec: + envoy: + xdsTransportProtocolVariant: GRPC networking: address: 192.168.0.1 inbound: diff --git a/pkg/plugins/runtime/k8s/metadata/annotations.go b/pkg/plugins/runtime/k8s/metadata/annotations.go index 959b6ef74609..3d0295f961fc 100644 --- a/pkg/plugins/runtime/k8s/metadata/annotations.go +++ b/pkg/plugins/runtime/k8s/metadata/annotations.go @@ -134,6 +134,10 @@ const ( // HeadlessService is "true" when the Service had ClusterIP: None, otherwise "false" HeadlessService = "k8s.kuma.io/is-headless-service" + + // KumaXdsTransportProtocolVariant allows to specify mode in which control-plane exchange configuration with the sidecar. + // Available values are: DELTA_GRPC and GRPC + KumaXdsTransportProtocolVariant = "kuma.io/xds-transport-protocol-variant" ) var PodAnnotationDeprecations = []Deprecation{ diff --git a/pkg/plugins/runtime/k8s/plugin.go b/pkg/plugins/runtime/k8s/plugin.go index f78a71588f70..0bc46e4137aa 100644 --- a/pkg/plugins/runtime/k8s/plugin.go +++ b/pkg/plugins/runtime/k8s/plugin.go @@ -205,6 +205,7 @@ func addPodReconciler(mgr kube_ctrl.Manager, rt core_runtime.Runtime, converter Mode: rt.Config().Mode, ResourceConverter: converter, KubeOutboundsAsVIPs: rt.Config().Experimental.KubeOutboundsAsVIPs, + DeltaXds: rt.Config().Experimental.DeltaXds, }, ResourceConverter: converter, SystemNamespace: rt.Config().Store.Kubernetes.SystemNamespace, diff --git a/pkg/xds/bootstrap/components.go b/pkg/xds/bootstrap/components.go index 7d793920f078..5eea7afc5a35 100644 --- a/pkg/xds/bootstrap/components.go +++ b/pkg/xds/bootstrap/components.go @@ -21,7 +21,7 @@ func RegisterBootstrap(rt core_runtime.Runtime) error { rt.Config().DpServer.Authn.EnableReloadableTokens || rt.Config().Store.Type == store.KubernetesStore, rt.Config().DpServer.Hds.Enabled, rt.Config().GetEnvoyAdminPort(), - rt.Config().Experimental.UseDeltaXds, + rt.Config().Experimental.DeltaXds, ) if err != nil { return err diff --git a/pkg/xds/bootstrap/generator.go b/pkg/xds/bootstrap/generator.go index 60264e6b1c8e..ec6129ceaf4a 100644 --- a/pkg/xds/bootstrap/generator.go +++ b/pkg/xds/bootstrap/generator.go @@ -127,12 +127,17 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr params.AdminPort = b.defaultAdminPort } } - switch request.XdsConfigMode { - case types.DELTA: - params.UseDelta = true - case types.NOT_DEFINED: - if b.deltaXdsEnabled { - params.UseDelta = true + setXdsTransportProtocolVariant := func(resourceMode mesh_proto.EnvoyConfiguration_XdsTransportProtocolVariant) { + switch resourceMode { + case mesh_proto.EnvoyConfiguration_DEFAULT: + if b.deltaXdsEnabled { + params.XdsTransportProtocolVariant = types.DELTA_GRPC + } + params.XdsTransportProtocolVariant = types.GRPC + case mesh_proto.EnvoyConfiguration_DELTA_GRPC: + params.XdsTransportProtocolVariant = types.DELTA_GRPC + case mesh_proto.EnvoyConfiguration_GRPC: + params.XdsTransportProtocolVariant = types.GRPC } } @@ -145,6 +150,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr params.Service = "ingress" setAdminPort(zoneIngress.Spec.GetNetworking().GetAdmin().GetPort()) + setXdsTransportProtocolVariant(zoneIngress.Spec.GetEnvoy().GetXdsTransportProtocolVariant()) case mesh_proto.EgressProxyType: zoneEgress, err := b.zoneEgressFor(ctx, request, proxyId) if err != nil { @@ -152,6 +158,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr } params.Service = "egress" setAdminPort(zoneEgress.Spec.GetNetworking().GetAdmin().GetPort()) + setXdsTransportProtocolVariant(zoneEgress.Spec.GetEnvoy().GetXdsTransportProtocolVariant()) case mesh_proto.DataplaneProxyType, "": params.HdsEnabled = b.hdsEnabled dataplane, err := b.dataplaneFor(ctx, request, proxyId) @@ -173,6 +180,7 @@ func (b *bootstrapGenerator) Generate(ctx context.Context, request types.Bootstr } params.Service = dataplane.Spec.GetIdentifyingService() setAdminPort(dataplane.Spec.GetNetworking().GetAdmin().GetPort()) + setXdsTransportProtocolVariant(dataplane.Spec.GetEnvoy().GetXdsTransportProtocolVariant()) err = b.getMetricsConfig(ctx, dataplane, &kumaDpBootstrap) if err != nil { diff --git a/pkg/xds/bootstrap/generator_test.go b/pkg/xds/bootstrap/generator_test.go index 1141fbb25086..c51e6c2caf23 100644 --- a/pkg/xds/bootstrap/generator_test.go +++ b/pkg/xds/bootstrap/generator_test.go @@ -224,11 +224,10 @@ var _ = Describe("bootstrapGenerator", func() { return dp }, request: types.BootstrapRequest{ - Mesh: "mesh", - Name: "name.namespace", - Version: defaultVersion, - Workdir: "/tmp", - XdsConfigMode: types.DELTA, + Mesh: "mesh", + Name: "name.namespace", + Version: defaultVersion, + Workdir: "/tmp", }, expectedConfigFile: "generator.custom-config-minimal-request-and-delta.golden.yaml", hdsEnabled: true, @@ -277,6 +276,9 @@ var _ = Describe("bootstrapGenerator", func() { ], "admin": { "port": 1234 + }, + "envoy" : { + "xdsTransportProtocolVariant": "DELTA_GRPC" } } }`, diff --git a/pkg/xds/bootstrap/parameters.go b/pkg/xds/bootstrap/parameters.go index ab84368499b4..eafe11bf9a2a 100644 --- a/pkg/xds/bootstrap/parameters.go +++ b/pkg/xds/bootstrap/parameters.go @@ -26,31 +26,31 @@ type AggregateMetricsConfig struct { } type configParameters struct { - Id string - Service string - AdminAddress string - AdminPort uint32 - ReadinessPort uint32 - AppProbeProxyEnabled bool - AdminAccessLogPath string - XdsHost string - XdsPort uint32 - XdsConnectTimeout time.Duration - Workdir string - MetricsCertPath string - MetricsKeyPath string - DataplaneToken string - DataplaneTokenPath string - DataplaneResource string - CertBytes []byte - Version *mesh_proto.Version - HdsEnabled bool - DynamicMetadata map[string]string - DNSPort uint32 - ProxyType string - Features []string - IsGatewayDataplane bool - Resources types.ProxyResources - SystemCaPath string - UseDelta bool + Id string + Service string + AdminAddress string + AdminPort uint32 + ReadinessPort uint32 + AppProbeProxyEnabled bool + AdminAccessLogPath string + XdsHost string + XdsPort uint32 + XdsConnectTimeout time.Duration + Workdir string + MetricsCertPath string + MetricsKeyPath string + DataplaneToken string + DataplaneTokenPath string + DataplaneResource string + CertBytes []byte + Version *mesh_proto.Version + HdsEnabled bool + DynamicMetadata map[string]string + DNSPort uint32 + ProxyType string + Features []string + IsGatewayDataplane bool + Resources types.ProxyResources + SystemCaPath string + XdsTransportProtocolVariant types.XdsMode } diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go index a84aae7e5d65..8d462a069b72 100644 --- a/pkg/xds/bootstrap/template_v3.go +++ b/pkg/xds/bootstrap/template_v3.go @@ -26,6 +26,7 @@ import ( "github.com/kumahq/kuma/pkg/config/xds" core_xds "github.com/kumahq/kuma/pkg/core/xds" util_proto "github.com/kumahq/kuma/pkg/util/proto" + "github.com/kumahq/kuma/pkg/xds/bootstrap/types" clusters_v3 "github.com/kumahq/kuma/pkg/xds/envoy/clusters/v3" "github.com/kumahq/kuma/pkg/xds/envoy/names" "github.com/kumahq/kuma/pkg/xds/envoy/tls" @@ -107,7 +108,8 @@ func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloada }) } configType := envoy_core_v3.ApiConfigSource_GRPC - if parameters.UseDelta { + switch parameters.XdsTransportProtocolVariant { + case types.DELTA_GRPC: configType = envoy_core_v3.ApiConfigSource_DELTA_GRPC } diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml index d813a9ebc47f..421ee94019d3 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config-minimal-request-and-delta.golden.yaml @@ -14,7 +14,7 @@ defaultRegexEngine: '@type': type.googleapis.com/envoy.extensions.regex_engines.v3.GoogleRE2 dynamicResources: adsConfig: - apiType: DELTA_GRPC + apiType: GRPC grpcServices: - envoyGrpc: clusterName: ads_cluster diff --git a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml index 4273701c3530..4c6fd737c830 100644 --- a/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml +++ b/pkg/xds/bootstrap/testdata/generator.custom-config.golden.yaml @@ -76,6 +76,9 @@ node: ], "admin": { "port": 1234 + }, + "envoy" : { + "xdsTransportProtocolVariant": "DELTA_GRPC" } } } diff --git a/pkg/xds/bootstrap/types/bootstrap_request.go b/pkg/xds/bootstrap/types/bootstrap_request.go index 6fbdb094d9ab..042eccdd9dac 100644 --- a/pkg/xds/bootstrap/types/bootstrap_request.go +++ b/pkg/xds/bootstrap/types/bootstrap_request.go @@ -21,15 +21,14 @@ type BootstrapRequest struct { Workdir string `json:"workdir"` MetricsResources MetricsResources `json:"metricsResources"` SystemCaPath string `json:"systemCaPath"` - XdsConfigMode XdsMode `json:"xdsConfigMode,omitempty"` } -type XdsMode int +type XdsMode string const ( - NOT_DEFINED XdsMode = iota - SOTW - DELTA + DEFAULT XdsMode = "DEFAULT" + DELTA_GRPC XdsMode = "DELTA_GRPC" + GRPC XdsMode = "GRPC" ) type Version struct { diff --git a/test/framework/envs/multizone/env.go b/test/framework/envs/multizone/env.go index a1dada76f876..a402c6f4acc7 100644 --- a/test/framework/envs/multizone/env.go +++ b/test/framework/envs/multizone/env.go @@ -132,7 +132,7 @@ func SetupAndGetState() []byte { kubeZone2Options := append( framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.KubeZone2), - WithEnv("KUMA_EXPERIMENTAL_USE_DELTA_XDS", "true"), + WithEnv("KUMA_EXPERIMENTAL_DELTA_XDS", "true"), ) kubeZone2Options = append(kubeZone2Options, WithCNI()) KubeZone2 = setupKubeZone(&wg, Kuma2, kubeZone2Options...) From fe30989a54657bfb15a3705e33fdae035a309007 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Tue, 12 Nov 2024 11:07:29 +0100 Subject: [PATCH 10/12] fix and add error Signed-off-by: Lukasz Dziedziak --- pkg/config/loader_test.go | 5 ----- .../runtime/k8s/controllers/egress_converter.go | 5 ++++- .../runtime/k8s/controllers/envoy_configuration.go | 12 ++++++++---- .../runtime/k8s/controllers/ingress_converter.go | 5 ++++- pkg/plugins/runtime/k8s/controllers/pod_converter.go | 6 +++++- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/pkg/config/loader_test.go b/pkg/config/loader_test.go index 742049c6e2c1..d9ec5a03708d 100644 --- a/pkg/config/loader_test.go +++ b/pkg/config/loader_test.go @@ -370,12 +370,7 @@ var _ = Describe("Config loader", func() { Expect(cfg.Experimental.KDSEventBasedWatchdog.DelayFullResync).To(BeTrue()) Expect(cfg.Experimental.AutoReachableServices).To(BeTrue()) Expect(cfg.Experimental.SidecarContainers).To(BeTrue()) -<<<<<<< HEAD - Expect(cfg.Experimental.SkipPersistedVIPs).To(BeTrue()) - Expect(cfg.Experimental.GenerateMeshServices).To(BeTrue()) Expect(cfg.Experimental.DeltaXds).To(BeTrue()) -======= ->>>>>>> upstream/master Expect(cfg.Proxy.Gateway.GlobalDownstreamMaxConnections).To(BeNumerically("==", 1)) Expect(cfg.EventBus.BufferSize).To(Equal(uint(30))) diff --git a/pkg/plugins/runtime/k8s/controllers/egress_converter.go b/pkg/plugins/runtime/k8s/controllers/egress_converter.go index 36638ac9e817..879f39873305 100644 --- a/pkg/plugins/runtime/k8s/controllers/egress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/egress_converter.go @@ -42,7 +42,10 @@ func (p *PodConverter) EgressFor( if exist { zoneEgress.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } - zoneEgress.Envoy = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) + zoneEgress.Envoy, err = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) + if err != nil { + return err + } return nil } diff --git a/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go b/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go index 6d169196ae11..49de265cda17 100644 --- a/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go +++ b/pkg/plugins/runtime/k8s/controllers/envoy_configuration.go @@ -1,11 +1,13 @@ package controllers import ( + "github.com/pkg/errors" + mesh_proto "github.com/kumahq/kuma/api/mesh/v1alpha1" "github.com/kumahq/kuma/pkg/plugins/runtime/k8s/metadata" ) -func GetEnvoyConfiguration(deltaXds bool, annotations metadata.Annotations) *mesh_proto.EnvoyConfiguration { +func GetEnvoyConfiguration(deltaXds bool, annotations metadata.Annotations) (*mesh_proto.EnvoyConfiguration, error) { envoyConfig := &mesh_proto.EnvoyConfiguration{ XdsTransportProtocolVariant: mesh_proto.EnvoyConfiguration_GRPC, } @@ -15,11 +17,13 @@ func GetEnvoyConfiguration(deltaXds bool, annotations metadata.Annotations) *mes xdsTransportProtocolVariant, exist := annotations.GetString(metadata.KumaXdsTransportProtocolVariant) if exist { switch xdsTransportProtocolVariant { - case "DELTA_GRPC": + case mesh_proto.EnvoyConfiguration_DELTA_GRPC.String(): envoyConfig.XdsTransportProtocolVariant = mesh_proto.EnvoyConfiguration_DELTA_GRPC - case "GRPC": + case mesh_proto.EnvoyConfiguration_GRPC.String(): envoyConfig.XdsTransportProtocolVariant = mesh_proto.EnvoyConfiguration_GRPC + default: + return nil, errors.Errorf("invalid xds transport protocol variant '%s'", xdsTransportProtocolVariant) } } - return envoyConfig + return envoyConfig, nil } diff --git a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go index e4db7029cb7f..54063eeca790 100644 --- a/pkg/plugins/runtime/k8s/controllers/ingress_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/ingress_converter.go @@ -66,7 +66,10 @@ func (p *PodConverter) IngressFor( if exist { zoneIngress.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } - zoneIngress.Envoy = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) + zoneIngress.Envoy, err = GetEnvoyConfiguration(p.DeltaXds, metadata.Annotations(pod.Annotations)) + if err != nil { + return err + } return nil } diff --git a/pkg/plugins/runtime/k8s/controllers/pod_converter.go b/pkg/plugins/runtime/k8s/controllers/pod_converter.go index 5aba2d53d319..c7570911f985 100644 --- a/pkg/plugins/runtime/k8s/controllers/pod_converter.go +++ b/pkg/plugins/runtime/k8s/controllers/pod_converter.go @@ -301,7 +301,11 @@ func (p *PodConverter) dataplaneFor( if exist { dataplane.Networking.Admin = &mesh_proto.EnvoyAdmin{Port: adminPort} } - dataplane.Envoy = GetEnvoyConfiguration(p.DeltaXds, annotations) + dataplane.Envoy, err = GetEnvoyConfiguration(p.DeltaXds, annotations) + if err != nil { + return nil, err + } + return dataplane, nil } From 9c57a2ec9080fafafc688ba9b15ebf2568602c62 Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Wed, 20 Nov 2024 17:07:50 +0100 Subject: [PATCH 11/12] review Signed-off-by: Lukasz Dziedziak --- pkg/hds/tracker/callbacks.go | 3 +- pkg/xds/auth/callbacks.go | 32 ++++---- .../server/callbacks/dataplane_callbacks.go | 73 ++++++------------- .../callbacks/dataplane_status_tracker.go | 66 +++++++---------- 4 files changed, 65 insertions(+), 109 deletions(-) diff --git a/pkg/hds/tracker/callbacks.go b/pkg/hds/tracker/callbacks.go index 2c5c75ca1374..1f723bb0819e 100644 --- a/pkg/hds/tracker/callbacks.go +++ b/pkg/hds/tracker/callbacks.go @@ -41,8 +41,7 @@ type tracker struct { sync.RWMutex // protects access to the fields below streamsAssociation map[xds.StreamID]core_model.ResourceKey - // deltaStreamsAssociation map[xds.StreamID]core_model.ResourceKey - dpStreams map[core_model.ResourceKey]streams + dpStreams map[core_model.ResourceKey]streams } func NewCallbacks( diff --git a/pkg/xds/auth/callbacks.go b/pkg/xds/auth/callbacks.go index 2666310567ee..55bf155e89de 100644 --- a/pkg/xds/auth/callbacks.go +++ b/pkg/xds/auth/callbacks.go @@ -56,6 +56,14 @@ type authCallbacks struct { deltaStreams map[core_xds.StreamID]stream } +func (d *authCallbacks) getStreams() map[core_xds.StreamID]stream { + return d.streams +} + +func (d *authCallbacks) getDeltaStreams() map[core_xds.StreamID]stream { + return d.deltaStreams +} + type stream struct { // context of the stream that contains a credential ctx context.Context @@ -85,7 +93,7 @@ func (a *authCallbacks) OnStreamClosed(streamID core_xds.StreamID) { } func (a *authCallbacks) OnStreamRequest(streamID core_xds.StreamID, req util_xds.DiscoveryRequest) error { - return a.onStreamRequest(streamID, req, false) + return a.onStreamRequest(streamID, req, a.getStreams) } func (a *authCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { @@ -108,11 +116,11 @@ func (a *authCallbacks) OnDeltaStreamClosed(streamID int64) { } func (a *authCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, req util_xds.DeltaDiscoveryRequest) error { - return a.onStreamRequest(streamID, req, true) + return a.onStreamRequest(streamID, req, a.getDeltaStreams) } -func (a *authCallbacks) onStreamRequest(streamID core_xds.StreamID, req util_xds.Request, isDelta bool) error { - s, err := a.stream(streamID, req, isDelta) +func (a *authCallbacks) onStreamRequest(streamID core_xds.StreamID, req util_xds.Request, getStreamsState func() map[core_xds.StreamID]stream) error { + s, err := a.stream(streamID, req, getStreamsState) if err != nil { return err } @@ -126,24 +134,14 @@ func (a *authCallbacks) onStreamRequest(streamID core_xds.StreamID, req util_xds return errors.Wrap(err, "authentication failed") } a.Lock() - if isDelta { - a.deltaStreams[streamID] = s - } else { - a.streams[streamID] = s - } + getStreamsState()[streamID] = s a.Unlock() return nil } -func (a *authCallbacks) stream(streamID core_xds.StreamID, req util_xds.Request, isDelta bool) (stream, error) { +func (a *authCallbacks) stream(streamID core_xds.StreamID, req util_xds.Request, getStreamsState func() map[core_xds.StreamID]stream) (stream, error) { a.RLock() - var s stream - var ok bool - if isDelta { - s, ok = a.deltaStreams[streamID] - } else { - s, ok = a.streams[streamID] - } + s, ok := getStreamsState()[streamID] a.RUnlock() if !ok { return stream{}, errors.New("stream is not present") diff --git a/pkg/xds/server/callbacks/dataplane_callbacks.go b/pkg/xds/server/callbacks/dataplane_callbacks.go index 59e88778f46c..a4b78b66ac2c 100644 --- a/pkg/xds/server/callbacks/dataplane_callbacks.go +++ b/pkg/xds/server/callbacks/dataplane_callbacks.go @@ -54,41 +54,44 @@ type dpStream struct { ctx context.Context } +func (d *xdsCallbacks) getDpStream() map[core_xds.StreamID]dpStream { + return d.dpStreams +} + +func (d *xdsCallbacks) getDpDeltaStream() map[core_xds.StreamID]dpStream { + return d.dpDeltaStreams +} + var _ util_xds.MultiXDSCallbacks = &xdsCallbacks{} func (d *xdsCallbacks) OnStreamClosed(streamID core_xds.StreamID) { - d.onStreamClosed(streamID, false) + d.onStreamClosed(streamID, d.getDpStream) } func (d *xdsCallbacks) OnDeltaStreamClosed(streamID core_xds.StreamID) { - d.onStreamClosed(streamID, true) + d.onStreamClosed(streamID, d.getDpDeltaStream) } func (d *xdsCallbacks) OnStreamRequest(streamID core_xds.StreamID, request util_xds.DiscoveryRequest) error { - return d.onStreamRequest(streamID, request, false) + return d.onStreamRequest(streamID, request, d.getDpStream) } func (d *xdsCallbacks) OnStreamDeltaRequest(streamID core_xds.StreamID, request util_xds.DeltaDiscoveryRequest) error { - return d.onStreamRequest(streamID, request, true) + return d.onStreamRequest(streamID, request, d.getDpDeltaStream) } func (d *xdsCallbacks) OnStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { - return d.onStreamOpen(ctx, streamID, false) + return d.onStreamOpen(ctx, streamID, d.getDpStream) } func (d *xdsCallbacks) OnDeltaStreamOpen(ctx context.Context, streamID core_xds.StreamID, _ string) error { - return d.onStreamOpen(ctx, streamID, true) + return d.onStreamOpen(ctx, streamID, d.getDpDeltaStream) } -func (d *xdsCallbacks) onStreamClosed(streamID core_xds.StreamID, isDelta bool) { +func (d *xdsCallbacks) onStreamClosed(streamID core_xds.StreamID, getDpStream func() map[core_xds.StreamID]dpStream) { var lastStreamDpKey *core_model.ResourceKey d.Lock() - var dpStream dpStream - if isDelta { - dpStream = d.dpDeltaStreams[streamID] - } else { - dpStream = d.dpStreams[streamID] - } + dpStream := getDpStream()[streamID] if dpKey := dpStream.dp; dpKey != nil { d.activeStreams[*dpKey]-- if d.activeStreams[*dpKey] == 0 { @@ -96,11 +99,7 @@ func (d *xdsCallbacks) onStreamClosed(streamID core_xds.StreamID, isDelta bool) delete(d.activeStreams, *dpKey) } } - if isDelta { - delete(d.dpDeltaStreams, streamID) - } else { - delete(d.dpStreams, streamID) - } + delete(getDpStream(), streamID) d.Unlock() if lastStreamDpKey != nil { // execute callback after lock is freed, so heavy callback implementation won't block every callback for every DPP. @@ -108,7 +107,7 @@ func (d *xdsCallbacks) onStreamClosed(streamID core_xds.StreamID, isDelta bool) } } -func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_xds.Request, isDelta bool) error { +func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_xds.Request, getDpStream func() map[core_xds.StreamID]dpStream) error { if request.NodeId() == "" { // from https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-versioning: // Only the first request on a stream is guaranteed to carry the node identifier. @@ -120,13 +119,7 @@ func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_ } d.RLock() - var alreadyProcessed bool - if isDelta { - alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil - } else { - alreadyProcessed = d.dpStreams[streamID].dp != nil - } - + alreadyProcessed := getDpStream()[streamID].dp != nil d.RUnlock() if alreadyProcessed { return nil @@ -146,27 +139,14 @@ func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_ // in case client will open 2 concurrent request for the same streamID then // we don't to increment the counter twice, so checking once again that stream // wasn't processed - if isDelta { - alreadyProcessed = d.dpDeltaStreams[streamID].dp != nil - } else { - alreadyProcessed = d.dpStreams[streamID].dp != nil - } + alreadyProcessed = getDpStream()[streamID].dp != nil if alreadyProcessed { return nil } - var dpStream dpStream - if isDelta { - dpStream = d.dpDeltaStreams[streamID] - } else { - dpStream = d.dpStreams[streamID] - } + dpStream := getDpStream()[streamID] dpStream.dp = &dpKey - if isDelta { - d.dpDeltaStreams[streamID] = dpStream - } else { - d.dpStreams[streamID] = dpStream - } + getDpStream()[streamID] = dpStream activeStreams := d.activeStreams[dpKey] d.activeStreams[dpKey]++ @@ -184,18 +164,13 @@ func (d *xdsCallbacks) onStreamRequest(streamID core_xds.StreamID, request util_ return nil } -func (d *xdsCallbacks) onStreamOpen(ctx context.Context, streamID core_xds.StreamID, isDelta bool) error { +func (d *xdsCallbacks) onStreamOpen(ctx context.Context, streamID core_xds.StreamID, getDpStream func() map[core_xds.StreamID]dpStream) error { d.Lock() defer d.Unlock() dps := dpStream{ ctx: ctx, } - if isDelta { - d.dpDeltaStreams[streamID] = dps - } else { - d.dpStreams[streamID] = dps - } - + getDpStream()[streamID] = dps return nil } diff --git a/pkg/xds/server/callbacks/dataplane_status_tracker.go b/pkg/xds/server/callbacks/dataplane_status_tracker.go index bc23961c9514..ea95cabe50f4 100644 --- a/pkg/xds/server/callbacks/dataplane_status_tracker.go +++ b/pkg/xds/server/callbacks/dataplane_status_tracker.go @@ -54,6 +54,14 @@ type dataplaneStatusTracker struct { deltaStreams map[int64]*streamState } +func (d *dataplaneStatusTracker) getStreamsState() map[int64]*streamState { + return d.streams +} + +func (d *dataplaneStatusTracker) getDeltaStreamsState() map[int64]*streamState { + return d.deltaStreams +} + type streamState struct { stop chan struct{} // is used for stopping a goroutine that flushes Dataplane status periodically mu sync.RWMutex // protects access to the fields below @@ -64,45 +72,45 @@ type streamState struct { // OnStreamOpen is called once an xDS stream is open with a stream ID and the type URL (or "" for ADS). // Returning an error will end processing and close the stream. OnStreamClosed will still be called. func (c *dataplaneStatusTracker) OnStreamOpen(ctx context.Context, streamID int64, typ string) error { - return c.onStreamOpen(streamID, typ, false) + return c.onStreamOpen(streamID, typ, c.getStreamsState) } // OnDeltaStreamOpen is called once an Delta xDS stream is open with a stream ID and the type URL (or "" for ADS). // Returning an error will end processing and close the stream. OnDeltaStreamOpen will still be called. func (c *dataplaneStatusTracker) OnDeltaStreamOpen(_ context.Context, streamID int64, typ string) error { - return c.onStreamOpen(streamID, typ, true) + return c.onStreamOpen(streamID, typ, c.getDeltaStreamsState) } // OnStreamClosed is called immediately prior to closing an xDS stream with a stream ID. func (c *dataplaneStatusTracker) OnStreamClosed(streamID int64) { - c.onStreamClose(streamID, false) + c.onStreamClose(streamID, c.getStreamsState) } // OnDeltaStreamClosed is called immediately prior to closing an Delta xDS stream with a stream ID. func (c *dataplaneStatusTracker) OnDeltaStreamClosed(streamID int64) { - c.onStreamClose(streamID, true) + c.onStreamClose(streamID, c.getDeltaStreamsState) } // OnStreamRequest is called once a request is received on a stream. // Returning an error will end processing and close the stream. OnStreamClosed will still be called. func (c *dataplaneStatusTracker) OnStreamRequest(streamID int64, req util_xds.DiscoveryRequest) error { - return c.onStreamRequest(streamID, req, false) + return c.onStreamRequest(streamID, req, c.getStreamsState) } // OnStreamDeltaRequest is called once a request is received on a delta stream. // Returning an error will end processing and close the stream. OnStreamDeltaRequest will still be called. func (c *dataplaneStatusTracker) OnStreamDeltaRequest(streamID int64, req util_xds.DeltaDiscoveryRequest) error { - return c.onStreamRequest(streamID, req, true) + return c.onStreamRequest(streamID, req, c.getDeltaStreamsState) } // OnStreamResponse is called immediately prior to sending a response on a stream. func (c *dataplaneStatusTracker) OnStreamResponse(streamID int64, req util_xds.DiscoveryRequest, resp util_xds.DiscoveryResponse) { - c.onStreamResponse(streamID, req, resp, false) + c.onStreamResponse(streamID, req, resp, c.getStreamsState) } // OnStreamDeltaResponse is called immediately prior to sending a response on a delta stream. func (c *dataplaneStatusTracker) OnStreamDeltaResponse(streamID int64, req util_xds.DeltaDiscoveryRequest, resp util_xds.DeltaDiscoveryResponse) { - c.onStreamResponse(streamID, req, resp, true) + c.onStreamResponse(streamID, req, resp, c.getDeltaStreamsState) } // To keep logs short, we want to log "Listeners" instead of full qualified Envoy type url name @@ -131,16 +139,11 @@ func (s *streamState) Close() { close(s.stop) } -func (c *dataplaneStatusTracker) onStreamRequest(streamID int64, req util_xds.Request, isDelta bool) error { +func (c *dataplaneStatusTracker) onStreamRequest(streamID int64, req util_xds.Request, getStreamsState func() map[int64]*streamState) error { c.mu.RLock() // read access to the map of all ADS streams defer c.mu.RUnlock() - var state *streamState - if isDelta { - state = c.deltaStreams[streamID] - } else { - state = c.streams[streamID] - } + state := getStreamsState()[streamID] state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() @@ -228,16 +231,11 @@ func (c *dataplaneStatusTracker) onStreamRequest(streamID int64, req util_xds.Re return nil } -func (c *dataplaneStatusTracker) onStreamResponse(streamID int64, req util_xds.Request, resp util_xds.Response, isDelta bool) { +func (c *dataplaneStatusTracker) onStreamResponse(streamID int64, req util_xds.Request, resp util_xds.Response, getStreamsState func() map[int64]*streamState) { c.mu.RLock() // read access to the map of all ADS streams defer c.mu.RUnlock() - var state *streamState - if isDelta { - state = c.deltaStreams[streamID] - } else { - state = c.streams[streamID] - } + state := getStreamsState()[streamID] state.mu.Lock() // write access to the per Dataplane info defer state.mu.Unlock() @@ -267,7 +265,7 @@ func (c *dataplaneStatusTracker) onStreamResponse(streamID int64, req util_xds.R log.V(1).Info("config sent") } -func (c *dataplaneStatusTracker) onStreamOpen(streamID int64, typ string, isDelta bool) error { +func (c *dataplaneStatusTracker) onStreamOpen(streamID int64, typ string, getStreamsState func() map[int64]*streamState) error { c.mu.Lock() // write access to the map of all ADS streams defer c.mu.Unlock() @@ -286,36 +284,22 @@ func (c *dataplaneStatusTracker) onStreamOpen(streamID int64, typ string, isDelt subscription: subscription, } // save - if isDelta { - c.deltaStreams[streamID] = state - } else { - c.streams[streamID] = state - } + getStreamsState()[streamID] = state statusTrackerLog.V(1).Info("proxy connecting", "streamID", streamID, "type", typ, "subscriptionID", subscription.Id) return nil } -func (c *dataplaneStatusTracker) onStreamClose(streamID int64, isDelta bool) { +func (c *dataplaneStatusTracker) onStreamClose(streamID int64, getStreamsState func() map[int64]*streamState) { c.mu.Lock() // write access to the map of all ADS streams defer c.mu.Unlock() - var state *streamState - if isDelta { - state = c.deltaStreams[streamID] - } else { - state = c.streams[streamID] - } + state := getStreamsState()[streamID] if state == nil { statusTrackerLog.Info("[WARNING] proxy disconnected but no state in the status_tracker", "streamID", streamID) return } - - if isDelta { - delete(c.deltaStreams, streamID) - } else { - delete(c.streams, streamID) - } + delete(getStreamsState(), streamID) // finilize subscription state.mu.Lock() // write access to the per Dataplane info subscription := state.subscription From 4f3f1aa610dc8c2e33efcf2c1a5a6c0ab070967d Mon Sep 17 00:00:00 2001 From: Lukasz Dziedziak Date: Fri, 22 Nov 2024 10:51:13 +0100 Subject: [PATCH 12/12] test eds fix in go-control-plane Signed-off-by: Lukasz Dziedziak --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 07abcee65ab6..8ee93fb17292 100644 --- a/go.mod +++ b/go.mod @@ -232,4 +232,4 @@ require ( ) // remove once https://github.com/envoyproxy/go-control-plane/issues/875 is resolved -replace github.com/envoyproxy/go-control-plane => github.com/kumahq/go-control-plane v0.13.1-kong-1 +replace github.com/envoyproxy/go-control-plane => github.com/kumahq/go-control-plane v0.13.1-kong-2-warming-test diff --git a/go.sum b/go.sum index 3bfea4778f6c..974518fbb10e 100644 --- a/go.sum +++ b/go.sum @@ -231,8 +231,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kumahq/go-control-plane v0.13.1-kong-1 h1:y4VBsyImRrxfVA1/8Fk8QtDI18uo8upc0agVfg6x5qU= -github.com/kumahq/go-control-plane v0.13.1-kong-1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= +github.com/kumahq/go-control-plane v0.13.1-kong-2-warming-test h1:e5xDQL1olr+UzAA+HJXygU4zKE71yrgoDTNu1IbI8nU= +github.com/kumahq/go-control-plane v0.13.1-kong-2-warming-test/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=