Add crawler detect again to identify more bots

that are driving us nuts.

[#1421]

Gemfile

@@ -60,6 +60,7 @@ gem 'image_processing', '~> 1.2'
 gem 'rack-timeout'
 gem 'pg_search'
 
+gem 'crawler_detect'
 gem 'rack-attack'
 
 # downgrade gem to solve parsing error https://stackoverflow.com/questions/74725359/ruby-on-rails-legacy-application-update-generates-gem-psych-alias-error-psychb

Gemfile.lock

@@ -142,6 +142,8 @@ GEM
     country_select (10.0.0)
       countries (> 5.0, < 8.0)
     crass (1.0.6)
+    crawler_detect (1.2.4)
+      qonfig (>= 0.24)
     dalli (3.2.8)
     database_cleaner (1.7.0)
     date (3.4.0)
@@ -313,6 +315,7 @@ GEM
       method_source (~> 1.0)
     psych (3.3.4)
     public_suffix (6.0.1)
+    qonfig (0.29.0)
     racc (1.8.1)
     rack (2.2.10)
     rack-attack (6.7.0)
@@ -469,6 +472,7 @@ DEPENDENCIES
   byebug
   capybara (~> 3.38)
   country_select
+  crawler_detect
   dalli
   database_cleaner (~> 1.7.0)
   derailed_benchmarks

app/controllers/application_controller.rb

@@ -9,6 +9,7 @@ class ApplicationController < ActionController::Base
   before_action :set_current_region
   before_action :set_search_region
   before_action :check_cookie_consent
+  before_action :log_bot_activity
 
   def authenticate_admin!
     return if current_profile&.admin?
@@ -57,6 +58,13 @@ def validate_region(region)
   helper_method :search_region
 
   private
+  def log_bot_activity
+    return if Rails.env.test?
+    return unless request.is_crawler?
+
+    logger.warn "Crawler was here: #{request.crawler_name}"
+  end
+
   def build_missing_translations(object)
     I18n.available_locales.each do |locale|
       object.translations.build(locale: locale) unless object.translated_locales.include?(locale)

config/application.rb

@@ -30,5 +30,6 @@ class Application < Rails::Application
     # the framework and any gems in your application.
 
     config.exceptions_app = self.routes
+    config.middleware.use Rack::CrawlerDetect
   end
 end