Adds configs logging_add_custom_attributes, logging_drop_attribute_fb_input

pkg/config/config.go

@@ -904,6 +904,14 @@ type Config struct {
 	// Public: Yes
 	LoggingRetryLimit string `yaml:"logging_retry_limit" envconfig:"logging_retry_limit" public:"true"`
 
+	// LoggingAddCustomAtts adds custom_attributes to fluent-bit forwarder tags.
+	// Default: False
+	LoggingAddCustomAtts bool `yaml:"logging_add_custom_attributes" envconfig:"logging_add_custom_attributes" public:"false"`
+
+	// LoggingDropAttFbInput drops fb.input from fluent-bit forwarder tags.
+	// Default: False
+	LoggingDropAttFbInput bool `yaml:"logging_drop_attribute_fb_input" envconfig:"logging_drop_attribute_fb_input" public:"false"`
+
 	// FluentBitExePath is the location from where the agent can execute fluent-bit.
 	// Default (Linux): /opt/td-agent-bit/bin/td-agent-bit
 	// Default (Windows): C:\Program Files\New Relic\newrelic-infra\newrelic-integrations\logging\fluent-bit
@@ -1511,6 +1519,8 @@ type LogForward struct {
 	ProxyCfg         LogForwardProxy
 	RetryLimit       string
 	FluentBitVerbose bool
+	CommonAttributes CustomAttributeMap
+	DropAttFbInput   bool
 }
 
 type LogForwardProxy struct {
@@ -1523,22 +1533,31 @@ type LogForwardProxy struct {
 
 // NewLogForward creates a valid log forwarder config.
 func NewLogForward(config *Config, troubleshoot Troubleshoot) LogForward {
+
+	// Add any config specified common attributes for logs
+	var commonAttributes CustomAttributeMap
+	if config.LoggingAddCustomAtts {
+		commonAttributes = config.CustomAttributes
+	}
+
 	return LogForward{
-		Troubleshoot:     troubleshoot,
-		ConfigsDir:       config.LoggingConfigsDir,
-		HomeDir:          config.LoggingHomeDir,
-		License:          config.License,
-		IsFedramp:        config.Fedramp,
-		IsStaging:        config.Staging,
-		RetryLimit:       config.LoggingRetryLimit,
-		FluentBitVerbose: config.Log.Level == LogLevelTrace && config.Log.HasIncludeFilter(TracesFieldName, SupervisorTrace),
+		Troubleshoot: troubleshoot,
+		ConfigsDir:   config.LoggingConfigsDir,
+		HomeDir:      config.LoggingHomeDir,
+		License:      config.License,
+		IsFedramp:    config.Fedramp,
+		IsStaging:    config.Staging,
 		ProxyCfg: LogForwardProxy{
 			IgnoreSystemProxy: config.IgnoreSystemProxy,
 			Proxy:             config.Proxy,
 			CABundleFile:      config.CABundleFile,
 			CABundleDir:       config.CABundleDir,
 			ValidateCerts:     config.ProxyValidateCerts,
 		},
+		RetryLimit:       config.LoggingRetryLimit,
+		FluentBitVerbose: config.Log.Level == LogLevelTrace && config.Log.HasIncludeFilter(TracesFieldName, SupervisorTrace),
+		CommonAttributes: commonAttributes,
+		DropAttFbInput:   config.LoggingDropAttFbInput,
 	}
 }
 
@@ -1827,6 +1846,8 @@ func NewConfig() *Config {
 		DebugLogSec:                   defaultDebugLogSec,
 		TruncTextValues:               defaultTruncTextValues,
 		LogFormat:                     defaultLogFormat,
+		LoggingAddCustomAtts:          defaultLoggingAddCustomAtts,
+		LoggingDropAttFbInput:         defaultLoggingDropAttFbInput,
 		LoggingRetryLimit:             defaultLoggingRetryLimit,
 		HTTPServerHost:                defaultHTTPServerHost,
 		HTTPServerPort:                defaultHTTPServerPort,

pkg/config/defaults.go

@@ -79,6 +79,8 @@ var (
 	defaultLogFormat                     = LogFormatText
 	defaultLogLevel                      = LogLevelInfo
 	defaultLogForward                    = false
+	defaultLoggingAddCustomAtts          = false
+	defaultLoggingDropAttFbInput         = false
 	defaultLoggingRetryLimit             = "5"         // nolint:gochecknoglobals
 	defaultMaxInventorySize              = 1000 * 1000 // Size limit from Vortex collector service (1MB)
 	defaultPayloadCompressionLevel       = 6           // default compression level used in go, higher than this does not show tangible benefits

pkg/integrations/v4/logs/cfg.go

@@ -287,7 +287,17 @@ func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, ho
 			fb.Inputs = append(fb.Inputs, input)
 		}
 
-		fb.Filters = append(fb.Filters, filters...)
+		for _, filter := range filters {
+			_, ok := filter.Records["fb.input"]
+			if logFwdCfg.DropAttFbInput && ok {
+				// Delete fb.input if in filters
+				delete(filter.Records, "fb.input")
+			}
+			// If any records remain, add to filters
+			if len(filter.Records) > 0 {
+				fb.Filters = append(fb.Filters, filter)
+			}
+		}
 
 		if (external != FBCfgExternal{} && fb.ExternalCfg != FBCfgExternal{}) {
 			cfgLogger.Warn("External Fluent Bit configuration specified more than once. Only first one is considered, please remove any duplicates from the configuration.")
@@ -301,15 +311,26 @@ func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, ho
 	}
 
 	// This record_modifier FILTER adds common attributes for all the log records
-	fb.Filters = append(fb.Filters, FBCfgFilter{
+	commonFilter := FBCfgFilter{
 		Name:  fbFilterTypeRecordModifier,
 		Match: "*",
 		Records: map[string]string{
 			rAttEntityGUID: entityGUID,
 			rAttPluginType: logRecordModifierSource,
 			rAttHostname:   hostname,
 		},
-	})
+	}
+	// Add custom_attributes if provided
+	if logFwdCfg.CommonAttributes != nil {
+		for key, value := range logFwdCfg.CommonAttributes {
+			if !isReserved(key) {
+				commonFilter.Records[key] = fmt.Sprintf("%v", value)
+			} else {
+				cfgLogger.WithField("attribute", key).Warn("attribute name is a reserved keyword and will be ignored, please use a different name")
+			}
+		}
+	}
+	fb.Filters = append(fb.Filters, commonFilter)
 
 	// Newrelic OUTPUT plugin will send all the collected logs to Vortex
 	fb.Output = newNROutput(logFwdCfg)