From 7dca474acf1c6a5ffb7948b585c541a793691956 Mon Sep 17 00:00:00 2001
From: zaziemo <maren.heltsche@gmail.com>
Date: Mon, 4 Nov 2024 09:37:19 +0100
Subject: [PATCH] Add crawler detect again to identify more bots

that are driving us nuts.

[#1421]
---
 Gemfile                                   | 1 +
 Gemfile.lock                              | 4 ++++
 app/controllers/application_controller.rb | 8 ++++++++
 config/application.rb                     | 1 +
 4 files changed, 14 insertions(+)

diff --git a/Gemfile b/Gemfile
index e26d9d626..fa2fb6b2e 100644
--- a/Gemfile
+++ b/Gemfile
@@ -60,6 +60,7 @@ gem 'image_processing', '~> 1.2'
 gem 'rack-timeout'
 gem 'pg_search'
 
+gem 'crawler_detect'
 gem 'rack-attack'
 
 # downgrade gem to solve parsing error https://stackoverflow.com/questions/74725359/ruby-on-rails-legacy-application-update-generates-gem-psych-alias-error-psychb
diff --git a/Gemfile.lock b/Gemfile.lock
index fbef6b4b2..1530970d8 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -142,6 +142,8 @@ GEM
     country_select (10.0.0)
       countries (> 5.0, < 8.0)
     crass (1.0.6)
+    crawler_detect (1.2.4)
+      qonfig (>= 0.24)
     dalli (3.2.8)
     database_cleaner (1.7.0)
     date (3.4.0)
@@ -313,6 +315,7 @@ GEM
       method_source (~> 1.0)
     psych (3.3.4)
     public_suffix (6.0.1)
+    qonfig (0.29.0)
     racc (1.8.1)
     rack (2.2.10)
     rack-attack (6.7.0)
@@ -469,6 +472,7 @@ DEPENDENCIES
   byebug
   capybara (~> 3.38)
   country_select
+  crawler_detect
   dalli
   database_cleaner (~> 1.7.0)
   derailed_benchmarks
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 8b3e2006d..adeaffb69 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -9,6 +9,7 @@ class ApplicationController < ActionController::Base
   before_action :set_current_region
   before_action :set_search_region
   before_action :check_cookie_consent
+  before_action :log_bot_activity
 
   def authenticate_admin!
     return if current_profile&.admin?
@@ -57,6 +58,13 @@ def validate_region(region)
   helper_method :search_region
 
   private
+  def log_bot_activity
+    return if Rails.env.test?
+    return unless request.is_crawler?
+
+    logger.warn "Crawler was here: #{request.crawler_name}"
+  end
+
   def build_missing_translations(object)
     I18n.available_locales.each do |locale|
       object.translations.build(locale: locale) unless object.translated_locales.include?(locale)
diff --git a/config/application.rb b/config/application.rb
index 7a2c55662..372ed19c2 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -30,5 +30,6 @@ class Application < Rails::Application
     # the framework and any gems in your application.
 
     config.exceptions_app = self.routes
+    config.middleware.use Rack::CrawlerDetect
   end
 end