Add configuration options to apply labels to log events

CHANGELOG.md

@@ -2,11 +2,15 @@
 
 ## dev
 
-Version <dev> introduces instrumentation for the aws-sdk-lambda gem.
+Version <dev> introduces instrumentation for the aws-sdk-lambda gem and allows users to opt-in to adding labels to logs.
 
 - **Feature: Instrumentation for aws-sdk-lambda**
 
-  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926)
+  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926).
+
+- **Feature: Add new configuration options to attach custom tags (labels) to logs**
+
+  The Ruby agent now allows you to opt-in to adding your custom tags (labels) to agent-forwarded logs. With custom tags on logs, platform engineers can easily filter, search, and correlate log data for faster and more efficient troubleshooting, improved performance, and optimized resource utilization. [PR#2925](https://github.com/newrelic/newrelic-ruby-agent/pull/2925)
 
 ## v9.15.0
 

lib/new_relic/agent/configuration/default_source.rb

@@ -881,6 +881,21 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'A hash with key/value pairs to add as custom attributes to all log events forwarded to New Relic. If sending using an environment variable, the value must be formatted like: "key1=value1,key2=value2"'
         },
+        :'application_logging.forwarding.labels.enabled' => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, the agent attaches [labels](https://docs.newrelic.com/docs/apm/agents/ruby-agent/configuration/ruby-agent-configuration/#labels) to log records.'
+        },
+        :'application_logging.forwarding.labels.exclude' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :transform => DefaultSource.method(:convert_to_list),
+          :allowed_from_server => false,
+          :description => 'A case-insensitive array or comma-delimited string containing the labels to exclude from log records.'
+        },
         :'application_logging.forwarding.max_samples_stored' => {
           :default => 10000,
           :public => true,

lib/new_relic/agent/log_event_aggregator.rb

@@ -25,6 +25,7 @@ class LogEventAggregator < EventAggregator
       METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
       FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
       DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      LABELS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Labels/Ruby/%s'.freeze
       MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
 
       named :LogEventAggregator
@@ -38,6 +39,7 @@ class LogEventAggregator < EventAggregator
       METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
       FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
       DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+      LABELS_ENABLED_KEY = :'application_logging.forwarding.labels.enabled'
       LOG_LEVEL_KEY = :'application_logging.forwarding.log_level'
       CUSTOM_ATTRIBUTES_KEY = :'application_logging.forwarding.custom_attributes'
 
@@ -51,6 +53,7 @@ def initialize(events)
         @high_security = NewRelic::Agent.config[:high_security]
         @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
         @attributes = NewRelic::Agent::LogEventAttributes.new
+
         register_for_done_configuring(events)
       end
 
@@ -186,6 +189,10 @@ def add_custom_attributes(custom_attributes)
         attributes.add_custom_attributes(custom_attributes)
       end
 
+      def labels
+        @labels ||= create_labels
+      end
+
       # Because our transmission format (MELT) is different than historical
       # agent payloads, extract the munging here to keep the service focused
       # on the general harvest + transmit instead of the format.
@@ -201,8 +208,9 @@ def self.payload_to_melt_format(data)
         # To save on unnecessary data transmission, trim the entity.type
         # sent by classic logs-in-context
         common_attributes.delete(ENTITY_TYPE_KEY)
-
-        common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)
+        aggregator = NewRelic::Agent.agent.log_event_aggregator
+        common_attributes.merge!(aggregator.attributes.custom_attributes)
+        common_attributes.merge!(aggregator.labels)
 
         _, items = data
         payload = [{
@@ -247,6 +255,7 @@ def register_for_done_configuring(events)
           record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
           record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
           record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+          record_configuration_metric(LABELS_SUPPORTABILITY_FORMAT, LABELS_ENABLED_KEY)
 
           add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
         end
@@ -327,6 +336,23 @@ def severity_too_low?(severity)
 
         Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
       end
+
+      def create_labels
+        return NewRelic::EMPTY_HASH unless NewRelic::Agent.config[LABELS_ENABLED_KEY]
+
+        downcased_exclusions = NewRelic::Agent.config[:'application_logging.forwarding.labels.exclude'].map(&:downcase)
+        log_labels = {}
+
+        NewRelic::Agent.config.parsed_labels.each do |parsed_label|
+          next if downcased_exclusions.include?(parsed_label['label_type'].downcase)
+
+          # labels are referred to as tags in the UI, so prefix the
+          # label-related attributes with 'tags.*'
+          log_labels["tags.#{parsed_label['label_type']}"] = parsed_label['label_value']
+        end
+
+        log_labels
+      end
     end
   end
 end

test/new_relic/agent/log_event_aggregator_test.rb

@@ -52,7 +52,8 @@ def test_records_enabled_metrics_on_startup
         LogEventAggregator::OVERALL_ENABLED_KEY => true,
         LogEventAggregator::METRICS_ENABLED_KEY => true,
         LogEventAggregator::FORWARDING_ENABLED_KEY => true,
-        LogEventAggregator::DECORATING_ENABLED_KEY => true
+        LogEventAggregator::DECORATING_ENABLED_KEY => true,
+        LogEventAggregator::LABELS_ENABLED_KEY => true
       ) do
         NewRelic::Agent.config.notify_server_source_added
 
@@ -61,7 +62,8 @@ def test_records_enabled_metrics_on_startup
           'Supportability/Logging/Ruby/LogStasher/enabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/enabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/enabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/enabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/enabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/enabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -72,7 +74,8 @@ def test_records_disabled_metrics_on_startup
         LogEventAggregator::OVERALL_ENABLED_KEY => false,
         LogEventAggregator::METRICS_ENABLED_KEY => false,
         LogEventAggregator::FORWARDING_ENABLED_KEY => false,
-        LogEventAggregator::DECORATING_ENABLED_KEY => false
+        LogEventAggregator::DECORATING_ENABLED_KEY => false,
+        LogEventAggregator::LABELS_ENABLED_KEY => false
       ) do
         NewRelic::Agent.config.notify_server_source_added
 
@@ -81,7 +84,8 @@ def test_records_disabled_metrics_on_startup
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -324,6 +328,35 @@ def test_records_metrics_on_harvest
       end
     end
 
+    def test_seen_and_sent_zeroed_out_on_reset!
+      with_config(CAPACITY_KEY => 3) do
+        3.times { @aggregator.record('Are you counting this?', 'DEBUG') }
+        @aggregator.harvest! # seen/sent are counted on harvest, this zeroes things out
+
+        assert_metrics_recorded_exclusive({
+          'Logging/lines' => {:call_count => 3},
+          'Logging/lines/DEBUG' => {:call_count => 3},
+          'Logging/Forwarding/Dropped' => {:call_count => 0},
+          'Supportability/Logging/Forwarding/Seen' => {:call_count => 3},
+          'Supportability/Logging/Forwarding/Sent' => {:call_count => 3}
+        },
+          :ignore_filter => %r{^Supportability/API/})
+
+        @aggregator.record('Are you counting this?', 'DEBUG')
+        @aggregator.reset! # set seen/sent to zero, throwing out the latest #record
+        @aggregator.harvest! # nothing new should be available to count, so use same numbers as before
+
+        assert_metrics_recorded_exclusive({
+          'Logging/lines' => {:call_count => 3},
+          'Logging/lines/DEBUG' => {:call_count => 3},
+          'Logging/Forwarding/Dropped' => {:call_count => 0},
+          'Supportability/Logging/Forwarding/Seen' => {:call_count => 3},
+          'Supportability/Logging/Forwarding/Sent' => {:call_count => 3}
+        },
+          :ignore_filter => %r{^Supportability/API/})
+      end
+    end
+
     def test_high_security_mode
       with_config(CAPACITY_KEY => 5, :high_security => true) do
         # We refresh the high security setting on this notification
@@ -343,14 +376,24 @@ def test_high_security_mode
           'Supportability/Logging/Ruby/LogStasher/enabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/enabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/enabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
     end
 
     def test_overall_disabled
-      with_config(LogEventAggregator::OVERALL_ENABLED_KEY => false) do
+      # set the overall enabled key to false
+      # put all other configs as true
+      with_config(
+        LogEventAggregator::OVERALL_ENABLED_KEY => false,
+        :'instrumentation.logger' => 'auto',
+        LogEventAggregator::METRICS_ENABLED_KEY => true,
+        LogEventAggregator::FORWARDING_ENABLED_KEY => true,
+        LogEventAggregator::DECORATING_ENABLED_KEY => true,
+        LogEventAggregator::LABELS_ENABLED_KEY => true
+      ) do
         # Refresh the value of @enabled on the LogEventAggregator
         NewRelic::Agent.config.notify_server_source_added
 
@@ -366,7 +409,8 @@ def test_overall_disabled
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -392,13 +436,18 @@ def test_overall_disabled_in_high_security_mode
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
     end
 
     def test_basic_conversion_to_melt_format
+      # If @labels was assigned in another test, it might leak into the common attributes
+      # forcibly remove them so that the melt format method has to freshly add them
+      @aggregator.remove_instance_variable(:@labels) if @aggregator.instance_variable_defined?(:@labels)
+
       LinkingMetadata.stubs(:append_service_linking_metadata).returns({
         'entity.guid' => 'GUID',
         'entity.name' => 'Hola'
@@ -693,5 +742,71 @@ def test_nil_when_record_logstasher_errors
         assert_empty results
       end
     end
+
+    def test_labels_added_as_common_attributes_when_enabled
+      config = {:labels => 'Server:One;Data Center:Primary', :'application_logging.forwarding.labels.enabled' => true}
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_not_added_as_common_attributes_when_disabled
+      config = {:labels => 'Server:One;Data Center:Primary', :'application_logging.forwarding.labels.enabled' => false}
+      expected_label_attributes = {}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_not_added_as_common_attributes_when_excluded
+      config = {
+        :labels => 'Server:One;Data Center:Primary;Fake:two',
+        :'application_logging.forwarding.labels.enabled' => true,
+        :'application_logging.forwarding.labels.exclude' => ['Fake']
+      }
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_excluded_only_for_case_insensitive_match
+      config = {
+        :labels => 'Server:One;Data Center:Primary;fake:two;Faker:three',
+        :'application_logging.forwarding.labels.enabled' => true,
+        :'application_logging.forwarding.labels.exclude' => ['Fake']
+      }
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary', 'tags.Faker' => 'three'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    private
+
+    def assert_labels(config, expected_attributes = {})
+      # we should only have one log event aggregator per agent instance
+      # simulate that by resetting @labels between tests
+      @aggregator.remove_instance_variable(:@labels) if @aggregator.instance_variable_defined?(:@labels)
+
+      with_config(config) do
+        LinkingMetadata.stub('append_service_linking_metadata', {'entity.guid' => 'GUID', 'entity.name' => 'Hola'}) do
+          log_data = [
+            {
+              events_seen: 0,
+              reservoir_size: 0
+            },
+            [
+              [{"priority": 1}, {"message": 'This is a mess'}]
+            ]
+          ]
+
+          payload, _ = LogEventAggregator.payload_to_melt_format(log_data)
+          expected = [{
+            common: {attributes: {'entity.guid' => 'GUID', 'entity.name' => 'Hola'}.merge!(expected_attributes)},
+            logs: [{"message": 'This is a mess'}]
+          }]
+
+          assert_equal(payload, expected)
+        end
+      end
+    end
   end
 end