Merge branch 'dev' into how_will_we_ever_explain_this

CHANGELOG.md

@@ -2,11 +2,15 @@
 
 ## dev
 
-Version <dev> introduces instrumentation for the aws-sdk-lambda gem and fixes a bug with explain plans on Rails 7.2+.
+Version <dev> introduces instrumentation for the aws-sdk-lambda gem, allows users to opt-in to adding labels to logs, and fixes a bug with explain plans on Rails 7.2+.
 
 - **Feature: Instrumentation for aws-sdk-lambda**
 
-  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926)
+  If the aws-sdk-lambda gem is present and used to invoke remote AWS Lambda functions, timing and error details for the invocations will be reported to New Relic. [PR#2926](https://github.com/newrelic/newrelic-ruby-agent/pull/2926).
+
+- **Feature: Add new configuration options to attach custom tags (labels) to logs**
+
+  The Ruby agent now allows you to opt-in to adding your custom tags (labels) to agent-forwarded logs. With custom tags on logs, platform engineers can easily filter, search, and correlate log data for faster and more efficient troubleshooting, improved performance, and optimized resource utilization. [PR#2925](https://github.com/newrelic/newrelic-ruby-agent/pull/2925)
 
 - **Bugfix: Record explain plan traces on Rails 7.2+**
 

lib/new_relic/agent/configuration/default_source.rb

@@ -881,6 +881,21 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'A hash with key/value pairs to add as custom attributes to all log events forwarded to New Relic. If sending using an environment variable, the value must be formatted like: "key1=value1,key2=value2"'
         },
+        :'application_logging.forwarding.labels.enabled' => {
+          :default => false,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, the agent attaches [labels](https://docs.newrelic.com/docs/apm/agents/ruby-agent/configuration/ruby-agent-configuration/#labels) to log records.'
+        },
+        :'application_logging.forwarding.labels.exclude' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :transform => DefaultSource.method(:convert_to_list),
+          :allowed_from_server => false,
+          :description => 'A case-insensitive array or comma-delimited string containing the labels to exclude from log records.'
+        },
         :'application_logging.forwarding.max_samples_stored' => {
           :default => 10000,
           :public => true,
@@ -2703,47 +2718,196 @@ def self.notify
           :description => 'Defines the endpoint URL for posting security-related data',
           :dynamic_name => true
         },
-        :'security.detection.rci.enabled' => {
-          :default => true,
+        :'security.application_info.port' => {
+          :default => nil,
+          :allow_nil => true,
+          :public => true,
+          :type => Integer,
+          :external => true,
+          :allowed_from_server => false,
+          :description => 'The port the application is listening on. This setting is mandatory for Passenger servers. Other servers are detected by default.'
+        },
+        :'security.exclude_from_iast_scan.api' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :external => true,
+          :allowed_from_server => true,
+          :transform => DefaultSource.method(:convert_to_list),
+          :description => 'Defines API paths the security agent should ignore in IAST scans. Accepts an array of regex patterns matching the URI to ignore. The regex pattern should provide a complete match for the URL without the endpoint. For example, `[".*account.*"], [".*/\api\/v1\/.*?\/login"]`'
+        },
+        :'security.exclude_from_iast_scan.http_request_parameters.header' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :external => true,
+          :allowed_from_server => true,
+          :transform => DefaultSource.method(:convert_to_list),
+          :description => 'An array of HTTP request headers the security agent should ignore in IAST scans. The array should specify a list of patterns matching the headers to ignore.'
+        },
+        :'security.exclude_from_iast_scan.http_request_parameters.query' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :external => true,
+          :allowed_from_server => true,
+          :transform => DefaultSource.method(:convert_to_list),
+          :description => 'An array of HTTP request query parameters the security agent should ignore in IAST scans. The array should specify a list of patterns matching the HTTP request query parameters to ignore.'
+        },
+        :'security.exclude_from_iast_scan.http_request_parameters.body' => {
+          :default => [],
+          :public => true,
+          :type => Array,
+          :external => true,
+          :allowed_from_server => true,
+          :transform => DefaultSource.method(:convert_to_list),
+          :description => 'An array of HTTP request body keys the security agent should ignore in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.insecure_settings' => {
+          :default => false,
           :external => true,
           :public => true,
           :type => Boolean,
           :allowed_from_server => false,
-          :description => 'If `true`, enables RCI (remote code injection) detection'
+          :description => 'If `true`, disables the detection of low-severity insecure settings (e.g., hash, crypto, cookie, random generators, trust boundary).'
         },
-        :'security.detection.rxss.enabled' => {
-          :default => true,
+        :'security.exclude_from_iast_scan.iast_detection_category.invalid_file_access' => {
+          :default => false,
           :external => true,
           :public => true,
           :type => Boolean,
           :allowed_from_server => false,
-          :description => 'If `true`, enables RXSS (reflected cross-site scripting) detection'
+          :description => 'If `true`, disables file operation-related IAST detections (File Access & Application integrity violation)'
         },
-        :'security.detection.deserialization.enabled' => {
-          :default => true,
+        :'security.exclude_from_iast_scan.iast_detection_category.sql_injection' => {
+          :default => false,
           :external => true,
           :public => true,
           :type => Boolean,
           :allowed_from_server => false,
-          :description => 'If `true`, enables deserialization detection'
+          :description => 'If `true`, disables SQL injection detection in IAST scans.'
         },
-        :'security.application_info.port' => {
-          :default => nil,
-          :allow_nil => true,
+        :'security.exclude_from_iast_scan.iast_detection_category.nosql_injection' => {
+          :default => false,
+          :external => true,
           :public => true,
-          :type => Integer,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables NOSQL injection detection in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.ldap_injection' => {
+          :default => false,
           :external => true,
+          :public => true,
+          :type => Boolean,
           :allowed_from_server => false,
-          :description => 'The port the application is listening on. This setting is mandatory for Passenger servers. Other servers should be detected by default.'
+          :description => 'If `true`, disables LDAP injection detection in IAST scans.'
         },
-        :'security.request.body_limit' => {
-          :default => 300,
-          :allow_nil => true,
+        :'security.exclude_from_iast_scan.iast_detection_category.javascript_injection' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables Javascript injection detection in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.command_injection' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables system command injection detection in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.xpath_injection' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables XPATH injection detection in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.ssrf' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables Sever-Side Request Forgery (SSRF) detection in IAST scans.'
+        },
+        :'security.exclude_from_iast_scan.iast_detection_category.rxss' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => false,
+          :description => 'If `true`, disables Reflected Cross-Site Scripting (RXSS) detection in IAST scans.'
+        },
+        :'security.scan_schedule.delay' => {
+          :default => 0,
+          :public => true,
+          :type => Integer,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'Specifies the delay time (in minutes) before the IAST scan begins after the application starts.'
+        },
+        :'security.scan_schedule.duration' => {
+          :default => 0,
           :public => true,
           :type => Integer,
           :external => true,
+          :allowed_from_server => true,
+          :description => 'Specifies the length of time (in minutes) that the IAST scan will run.'
+        },
+        :'security.scan_schedule.schedule' => {
+          :default => '',
+          :public => true,
+          :type => String,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'Specifies a cron expression that sets when the IAST scan should run.',
+          :dynamic_name => true
+        },
+        :'security.scan_schedule.always_sample_traces' => {
+          :default => false,
+          :external => true,
+          :public => true,
+          :type => Boolean,
           :allowed_from_server => false,
-          :description => 'Defines the request body limit to process in security events (in KB). The default value is 300, for 300KB.'
+          :description => 'If `true`, allows IAST to continuously gather trace data in the background. Collected data will be used by the security agent to perform an IAST scan at the scheduled time.'
+        },
+        :'security.scan_controllers.iast_scan_request_rate_limit' => {
+          :default => 3600,
+          :public => true,
+          :type => Integer,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'Sets the maximum number of HTTP requests allowed for the IAST scan per minute. Any Integer between 12 and 3600 is valid. The default value is 3600.'
+        },
+        :'security.scan_controllers.scan_instance_count' => {
+          :default => 0,
+          :public => true,
+          :type => Integer,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'The number of application instances for a specific entity on which IAST analysis is performed.'
+        },
+        :'security.scan_controllers.report_http_response_body' => {
+          :default => true,
+          :public => true,
+          :type => Boolean,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'If `true`, enables the sending of HTTP responses bodies. Disabling this also disables Reflected Cross-Site Scripting (RXSS) vulnerability detection.'
+        },
+        :'security.iast_test_identifier' => {
+          :default => nil,
+          :allow_nil => true,
+          :public => true,
+          :type => String,
+          :external => true,
+          :allowed_from_server => true,
+          :description => 'Unique test identifier when runnning IAST in CI/CD environment to differentiate between different test runs, e.g., a build number.'
         }
       }.freeze
       # rubocop:enable Metrics/CollectionLiteralLength

lib/new_relic/agent/log_event_aggregator.rb

@@ -25,6 +25,7 @@ class LogEventAggregator < EventAggregator
       METRICS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Metrics/Ruby/%s'.freeze
       FORWARDING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Forwarding/Ruby/%s'.freeze
       DECORATING_SUPPORTABILITY_FORMAT = 'Supportability/Logging/LocalDecorating/Ruby/%s'.freeze
+      LABELS_SUPPORTABILITY_FORMAT = 'Supportability/Logging/Labels/Ruby/%s'.freeze
       MAX_BYTES = 32768 # 32 * 1024 bytes (32 kibibytes)
 
       named :LogEventAggregator
@@ -38,6 +39,7 @@ class LogEventAggregator < EventAggregator
       METRICS_ENABLED_KEY = :'application_logging.metrics.enabled'
       FORWARDING_ENABLED_KEY = :'application_logging.forwarding.enabled'
       DECORATING_ENABLED_KEY = :'application_logging.local_decorating.enabled'
+      LABELS_ENABLED_KEY = :'application_logging.forwarding.labels.enabled'
       LOG_LEVEL_KEY = :'application_logging.forwarding.log_level'
       CUSTOM_ATTRIBUTES_KEY = :'application_logging.forwarding.custom_attributes'
 
@@ -51,6 +53,7 @@ def initialize(events)
         @high_security = NewRelic::Agent.config[:high_security]
         @instrumentation_logger_enabled = NewRelic::Agent::Instrumentation::Logger.enabled?
         @attributes = NewRelic::Agent::LogEventAttributes.new
+
         register_for_done_configuring(events)
       end
 
@@ -186,6 +189,10 @@ def add_custom_attributes(custom_attributes)
         attributes.add_custom_attributes(custom_attributes)
       end
 
+      def labels
+        @labels ||= create_labels
+      end
+
       # Because our transmission format (MELT) is different than historical
       # agent payloads, extract the munging here to keep the service focused
       # on the general harvest + transmit instead of the format.
@@ -201,8 +208,9 @@ def self.payload_to_melt_format(data)
         # To save on unnecessary data transmission, trim the entity.type
         # sent by classic logs-in-context
         common_attributes.delete(ENTITY_TYPE_KEY)
-
-        common_attributes.merge!(NewRelic::Agent.agent.log_event_aggregator.attributes.custom_attributes)
+        aggregator = NewRelic::Agent.agent.log_event_aggregator
+        common_attributes.merge!(aggregator.attributes.custom_attributes)
+        common_attributes.merge!(aggregator.labels)
 
         _, items = data
         payload = [{
@@ -247,6 +255,7 @@ def register_for_done_configuring(events)
           record_configuration_metric(METRICS_SUPPORTABILITY_FORMAT, METRICS_ENABLED_KEY)
           record_configuration_metric(FORWARDING_SUPPORTABILITY_FORMAT, FORWARDING_ENABLED_KEY)
           record_configuration_metric(DECORATING_SUPPORTABILITY_FORMAT, DECORATING_ENABLED_KEY)
+          record_configuration_metric(LABELS_SUPPORTABILITY_FORMAT, LABELS_ENABLED_KEY)
 
           add_custom_attributes(NewRelic::Agent.config[CUSTOM_ATTRIBUTES_KEY])
         end
@@ -327,6 +336,23 @@ def severity_too_low?(severity)
 
         Logger::Severity.const_get(severity_constant) < Logger::Severity.const_get(configured_log_level_constant)
       end
+
+      def create_labels
+        return NewRelic::EMPTY_HASH unless NewRelic::Agent.config[LABELS_ENABLED_KEY]
+
+        downcased_exclusions = NewRelic::Agent.config[:'application_logging.forwarding.labels.exclude'].map(&:downcase)
+        log_labels = {}
+
+        NewRelic::Agent.config.parsed_labels.each do |parsed_label|
+          next if downcased_exclusions.include?(parsed_label['label_type'].downcase)
+
+          # labels are referred to as tags in the UI, so prefix the
+          # label-related attributes with 'tags.*'
+          log_labels["tags.#{parsed_label['label_type']}"] = parsed_label['label_value']
+        end
+
+        log_labels
+      end
     end
   end
 end

newrelic.yml

@@ -958,26 +958,13 @@ common: &default_settings
   # Passenger servers. Other servers should be detected by default.
   # security.application_info.port: nil
 
-  # If true, enables deserialization detection
-  # security.detection.deserialization.enabled: true
-
-  # If true, enables RCI (remote code injection) detection
-  # security.detection.rci.enabled: true
-
-  # If true, enables RXSS (reflected cross-site scripting) detection
-  # security.detection.rxss.enabled: true
-
   # If true, the security agent is started (the agent runs in its event loop)
   # security.enabled: false
 
   # Defines the mode for the security agent to operate in. Currently only IAST is
   # supported
   # security.mode: IAST
 
-  # Defines the request body limit to process in security events (in KB). The
-  # default value is 300, for 300KB.
-  # security.request.body_limit: 300
-
   # Defines the endpoint URL for posting security-related data
   # security.validator_service_url: wss://csec.nr-data.net
 

test/environments/norails/Gemfile

@@ -12,12 +12,8 @@ gem 'rack-test', '< 0.8.0'
 
 gem 'newrelic_rpm', :path => '../../..'
 
-group :development do
-  if ENV['ENABLE_PRY']
-    gem 'pry', '~> 0.14.1'
-    gem 'pry-nav'
-  end
-end
+gem 'pry' if ENV['ENABLE_PRY']
+
 gem 'simplecov' if ENV['VERBOSE_TEST_OUTPUT']
 
 gem 'warning'

test/environments/rails40/Gemfile

@@ -23,11 +23,7 @@ end
 
 gem 'newrelic_rpm', path: '../../..'
 
-group :development do
-  if ENV['ENABLE_PRY']
-    gem 'pry', '~> 0.9.12'
-  end
-end
+gem 'pry' if ENV['ENABLE_PRY']
 
 gem 'warning'
 gem 'loofah', '~> 2.20.0' if RUBY_VERSION >= '2.4.0' && RUBY_VERSION < '2.5.0'