Add configuration options to apply labels to log events

CHANGELOG.md

@@ -2,11 +2,15 @@
 
 ## dev
 
-Version <dev> introduces instrumentation for the aws-sdk-lambda gem.
+Version <dev> introduces instrumentation for the aws-sdk-lambda gem and allows users to opt-in to adding labels to logs.
 
 - **Feature: Instrumentation for aws-sdk-lambda**
 
-  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926)
+  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926).
+
+- **Feature: Add new configuration options to attach custom tags (labels) to logs**
+
+  The Ruby agent now allows you to opt-in to adding your custom tags (labels) to agent-forwarded logs. With custom tags on logs, platform engineers can easily filter, search, and correlate log data for faster and more efficient troubleshooting, improved performance, and optimized resource utilization. To learn more about this feature see the documentation: </docs/logs/logs-context/Custom-tags-agent-forwarder-logs>. [PR#2925](https://github.com/newrelic/newrelic-ruby-agent/pull/2925)
 
 ## v9.15.0
 

lib/new_relic/agent/configuration/default_source.rb

@@ -881,6 +881,21 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'A hash with key/value pairs to add as custom attributes to all log events forwarded to New Relic. If sending using an environment variable, the value must be formatted like: "key1=value1,key2=value2"'
         },
+        :'application_logging.forwarding.labels.enabled' => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, the agent attaches [labels](https://docs.newrelic.com/docs/apm/agents/ruby-agent/configuration/ruby-agent-configuration/#labels) to log records.'
+        },
+        :'application_logging.forwarding.labels.exclude' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :transform => DefaultSource.method(:convert_to_list),
+          :allowed_from_server => false,
+          :description => 'A case-insensitive array or comma-delimited string containing the labels to exclude from log records.'
+        },
         :'application_logging.forwarding.max_samples_stored' => {
           :default => 10000,
           :public => true,

lib/new_relic/agent/log_event_aggregator.rb

@@ -25,6 +25,7 @@ class LogEventAggregator < EventAggregator
       METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
       FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
       DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      LABELS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Labels/Ruby/%s'.freeze
       MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
 
       named :LogEventAggregator
@@ -38,6 +39,7 @@ class LogEventAggregator < EventAggregator
       METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
       FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
       DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+      LABELS_ENABLED_KEY = :'application_logging.forwarding.labels.enabled'
       LOG_LEVEL_KEY = :'application_logging.forwarding.log_level'
       CUSTOM_ATTRIBUTES_KEY = :'application_logging.forwarding.custom_attributes'
 
@@ -51,6 +53,7 @@ def initialize(events)
         @high_security = NewRelic::Agent.config[:high_security]
         @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
         @attributes = NewRelic::Agent::LogEventAttributes.new
+
         register_for_done_configuring(events)
       end
 
@@ -186,6 +189,10 @@ def add_custom_attributes(custom_attributes)
         attributes.add_custom_attributes(custom_attributes)
       end
 
+      def labels
+        @labels ||= create_labels
+      end
+
       # Because our transmission format (MELT) is different than historical
       # agent payloads, extract the munging here to keep the service focused
       # on the general harvest + transmit instead of the format.
@@ -201,8 +208,9 @@ def self.payload_to_melt_format(data)
         # To save on unnecessary data transmission, trim the entity.type
         # sent by classic logs-in-context
         common_attributes.delete(ENTITY_TYPE_KEY)
-
-        common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)
+        aggregator = NewRelic::Agent.agent.log_event_aggregator
+        common_attributes.merge!(aggregator.attributes.custom_attributes)
+        common_attributes.merge!(aggregator.labels)
 
         _, items = data
         payload = [{
@@ -222,6 +230,7 @@ def reset!
         @counter_lock.synchronize do
           @seen = 0
           @seen_by_severity.clear
+          remove_instance_variable(:@labels) if instance_variable_defined?(:@labels)
         end
 
         super
@@ -247,6 +256,7 @@ def register_for_done_configuring(events)
           record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
           record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
           record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+          record_configuration_metric(LABELS_SUPPORTABILITY_FORMAT, LABELS_ENABLED_KEY)
 
           add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
         end
@@ -327,6 +337,23 @@ def severity_too_low?(severity)
 
         Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
       end
+
+      def create_labels
+        return NewRelic::EMPTY_HASH unless NewRelic::Agent.config[LABELS_ENABLED_KEY]
+
+        downcased_exclusions = NewRelic::Agent.config[:'application_logging.forwarding.labels.exclude'].map(&:downcase)
+        log_labels = {}
+
+        NewRelic::Agent.config.parsed_labels.each do |parsed_label|
+          next if downcased_exclusions.include?(parsed_label['label_type'].downcase)
+
+          # labels are referred to as tags in the UI, so prefix the
+          # label-related attributes with 'tags.*'
+          log_labels["tags.#{parsed_label['label_type']}"] = parsed_label['label_value']
+        end
+
+        log_labels
+      end
     end
   end
 end

test/new_relic/agent/log_event_aggregator_test.rb

@@ -52,7 +52,8 @@ def test_records_enabled_metrics_on_startup
         LogEventAggregator::OVERALL_ENABLED_KEY => true,
         LogEventAggregator::METRICS_ENABLED_KEY => true,
         LogEventAggregator::FORWARDING_ENABLED_KEY => true,
-        LogEventAggregator::DECORATING_ENABLED_KEY => true
+        LogEventAggregator::DECORATING_ENABLED_KEY => true,
+        LogEventAggregator::LABELS_ENABLED_KEY => true
       ) do
         NewRelic::Agent.config.notify_server_source_added
 
@@ -61,7 +62,8 @@ def test_records_enabled_metrics_on_startup
           'Supportability/Logging/Ruby/LogStasher/enabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/enabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/enabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/enabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/enabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/enabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -72,7 +74,8 @@ def test_records_disabled_metrics_on_startup
         LogEventAggregator::OVERALL_ENABLED_KEY => false,
         LogEventAggregator::METRICS_ENABLED_KEY => false,
         LogEventAggregator::FORWARDING_ENABLED_KEY => false,
-        LogEventAggregator::DECORATING_ENABLED_KEY => false
+        LogEventAggregator::DECORATING_ENABLED_KEY => false,
+        LogEventAggregator::LABELS_ENABLED_KEY => false
       ) do
         NewRelic::Agent.config.notify_server_source_added
 
@@ -81,7 +84,8 @@ def test_records_disabled_metrics_on_startup
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -324,6 +328,35 @@ def test_records_metrics_on_harvest
       end
     end
 
+    def test_seen_and_sent_zeroed_out_on_reset!
+      with_config(CAPACITY_KEY => 3) do
+        3.times { @aggregator.record('Are you counting this?', 'DEBUG') }
+        @aggregator.harvest! # seen/sent are counted on harvest, this zeroes things out
+
+        assert_metrics_recorded_exclusive({
+          'Logging/lines' => {:call_count => 3},
+          'Logging/lines/DEBUG' => {:call_count => 3},
+          'Logging/Forwarding/Dropped' => {:call_count => 0},
+          'Supportability/Logging/Forwarding/Seen' => {:call_count => 3},
+          'Supportability/Logging/Forwarding/Sent' => {:call_count => 3}
+        },
+          :ignore_filter => %r{^Supportability/API/})
+
+        @aggregator.record('Are you counting this?', 'DEBUG')
+        @aggregator.reset! # set seen/sent to zero, throwing out the latest #record
+        @aggregator.harvest! # nothing new should be available to count, so use same numbers as before
+
+        assert_metrics_recorded_exclusive({
+          'Logging/lines' => {:call_count => 3},
+          'Logging/lines/DEBUG' => {:call_count => 3}, # ?
+          'Logging/Forwarding/Dropped' => {:call_count => 0},
+          'Supportability/Logging/Forwarding/Seen' => {:call_count => 3}, # ?
+          'Supportability/Logging/Forwarding/Sent' => {:call_count => 3}
+        },
+          :ignore_filter => %r{^Supportability/API/})
+      end
+    end
+
     def test_high_security_mode
       with_config(CAPACITY_KEY => 5, :high_security => true) do
         # We refresh the high security setting on this notification
@@ -343,14 +376,24 @@ def test_high_security_mode
           'Supportability/Logging/Ruby/LogStasher/enabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/enabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/enabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
     end
 
     def test_overall_disabled
-      with_config(LogEventAggregator::OVERALL_ENABLED_KEY => false) do
+      # set the overall enabled key to false
+      # put all other configs as true
+      with_config(
+        LogEventAggregator::OVERALL_ENABLED_KEY => false,
+        :'instrumentation.logger' => 'auto',
+        LogEventAggregator::METRICS_ENABLED_KEY => true,
+        LogEventAggregator::FORWARDING_ENABLED_KEY => true,
+        LogEventAggregator::DECORATING_ENABLED_KEY => true,
+        LogEventAggregator::LABELS_ENABLED_KEY => true
+      ) do
         # Refresh the value of @enabled on the LogEventAggregator
         NewRelic::Agent.config.notify_server_source_added
 
@@ -366,7 +409,8 @@ def test_overall_disabled
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -392,7 +436,8 @@ def test_overall_disabled_in_high_security_mode
           'Supportability/Logging/Ruby/LogStasher/disabled' => {:call_count => 1},
           'Supportability/Logging/Metrics/Ruby/disabled' => {:call_count => 1},
           'Supportability/Logging/Forwarding/Ruby/disabled' => {:call_count => 1},
-          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1}
+          'Supportability/Logging/LocalDecorating/Ruby/disabled' => {:call_count => 1},
+          'Supportability/Logging/Labels/Ruby/disabled' => {:call_count => 1}
         },
           :ignore_filter => %r{^Supportability/API/})
       end
@@ -693,5 +738,81 @@ def test_nil_when_record_logstasher_errors
         assert_empty results
       end
     end
+
+    def test_labels_added_as_common_attributes_when_enabled
+      config = {:labels => 'Server:One;Data Center:Primary', :'application_logging.forwarding.labels.enabled' => true}
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_not_added_as_common_attributes_when_disabled
+      config = {:labels => 'Server:One;Data Center:Primary', :'application_logging.forwarding.labels.enabled' => false}
+      expected_label_attributes = {}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_not_added_as_common_attributes_when_excluded
+      config = {
+        :labels => 'Server:One;Data Center:Primary;Fake:two',
+        :'application_logging.forwarding.labels.enabled' => true,
+        :'application_logging.forwarding.labels.exclude' => ['Fake']
+      }
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_excluded_only_for_case_insensitive_match
+      config = {
+        :labels => 'Server:One;Data Center:Primary;fake:two;Faker:three',
+        :'application_logging.forwarding.labels.enabled' => true,
+        :'application_logging.forwarding.labels.exclude' => ['Fake']
+      }
+      expected_label_attributes = {'tags.Server' => 'One', 'tags.Data Center' => 'Primary', 'tags.Faker' => 'three'}
+
+      assert_labels(config, expected_label_attributes)
+    end
+
+    def test_labels_undefined_during_reset!
+      with_config(:'application_logging.forwarding.labels.enabled' => true, :labels => 'Server:One;Data Center:primary') do
+        assert_equal({'tags.Server' => 'One', 'tags.Data Center' => 'primary'}, @aggregator.labels)
+        @aggregator.reset!
+
+        refute @aggregator.instance_variable_defined?(:@labels)
+      end
+    end
+
+    private
+
+    def assert_labels(config, expected_attributes = {})
+      # we should only have one log event aggregator per agent instance
+      # simulate that by resetting @labels between tests
+      aggregator = NewRelic::Agent.agent.log_event_aggregator
+      aggregator.remove_instance_variable(:@labels) if aggregator.instance_variable_defined?(:@labels)
+
+      with_config(config) do
+        LinkingMetadata.stub('append_service_linking_metadata', {'entity.guid' => 'GUID', 'entity.name' => 'Hola'}) do
+          log_data = [
+            {
+              events_seen: 0,
+              reservoir_size: 0
+            },
+            [
+              [{"priority": 1}, {"message": 'This is a mess'}]
+            ]
+          ]
+
+          payload, _ = LogEventAggregator.payload_to_melt_format(log_data)
+          expected = [{
+            common: {attributes: {'entity.guid' => 'GUID', 'entity.name' => 'Hola'}.merge!(expected_attributes)},
+            logs: [{"message": 'This is a mess'}]
+          }]
+
+          assert_equal(payload, expected)
+        end
+      end
+    end
   end
 end