Add member export and fix system tests (#1114)

# What it does

Adds a member and membership export to the admin UI.

.github/workflows/ci.yml

@@ -48,7 +48,7 @@ jobs:
         with:
           bundler-cache: true
 
-      - uses: nanasess/setup-chromedriver@v1
+      - uses: nanasess/setup-chromedriver@v2
         if: matrix.test == 'system'
 
       - name: Setup fonts

Gemfile

@@ -86,7 +86,6 @@ group :test do
   gem "minitest", "5.15.0"
   gem "capybara", ">= 2.15"
   gem "selenium-webdriver"
-  gem "webdrivers", "~> 5.0"
   gem "minitest-ci"
   gem "rails-controller-testing"
 end

Gemfile.lock

@@ -253,7 +253,7 @@ GEM
     method_source (1.0.0)
     mini_magick (4.11.0)
     mini_mime (1.1.2)
-    mini_portile2 (2.8.2)
+    mini_portile2 (2.8.4)
     minitest (5.15.0)
     minitest-ci (3.4.0)
       minitest (>= 5.0.6)
@@ -376,7 +376,7 @@ GEM
       railties (>= 5.0)
     reverse_markdown (2.1.1)
       nokogiri
-    rexml (3.2.5)
+    rexml (3.2.6)
     rubocop (1.27.0)
       parallel (~> 1.10)
       parser (>= 3.1.0.0)
@@ -399,7 +399,7 @@ GEM
     scenic (1.6.0)
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
-    selenium-webdriver (4.10.0)
+    selenium-webdriver (4.11.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
@@ -472,10 +472,6 @@ GEM
       activemodel (>= 6.0.0)
       bindex (>= 0.4.0)
       railties (>= 6.0.0)
-    webdrivers (5.2.0)
-      nokogiri (~> 1.6)
-      rubyzip (>= 1.3.0)
-      selenium-webdriver (~> 4.0)
     webpacker (5.4.4)
       activesupport (>= 5.2)
       rack-proxy (>= 0.6.1)
@@ -545,7 +541,6 @@ DEPENDENCIES
   turbolinks (~> 5)
   turbolinks_render
   web-console (>= 3.3.0)
-  webdrivers (~> 5.0)
   webpacker (= 5.4.4)
 
 RUBY VERSION

app/controllers/admin/settings/exports_controller.rb

@@ -4,18 +4,29 @@ class Admin::Settings::ExportsController < Admin::BaseController
   def index
   end
 
-  def create
+  def items
     now = Time.current.rfc3339
     filename = "all-items-#{now}.csv"
+    exporter = ItemExporter.new(Item.all)
+    export(filename, exporter)
+  end
+
+  def members
+    now = Time.current.rfc3339
+    filename = "all-members-and-memberships-#{now}.csv"
+    exporter = MemberExporter.new
+    export(filename, exporter)
+  end
+
+  private
 
+  def export(filename, exporter)
     send_file_headers!(
       type: "text/csv",
       disposition: "attachment",
       filename: filename
     )
     response.headers["Last-Modified"] = Time.now.httpdate
-
-    exporter = ItemExporter.new(Item.all)
     exporter.export(response.stream)
   ensure
     response.stream.close

app/lib/member_exporter.rb

@@ -0,0 +1,72 @@
+require "csv"
+
+class MemberExporter
+  def initialize
+    @members = Member.for_export
+    @year_range = Membership.year_range_for_export
+  end
+
+  def export(stream)
+    headers = [
+      "circulate_id",
+      "preferred_name",
+      "legal_name",
+      "email",
+      "phone",
+      "status",
+      "address1",
+      "address2",
+      "city",
+      "region",
+      "postal_code",
+      "number",
+      "pronouns",
+      "volunteer_interest",
+      *year_headers
+    ]
+
+    stream.write(CSV.generate_line(headers))
+
+    @members.find_each(batch_size: 250) do |member|
+      # 16 rows
+      row = [
+        member.id,
+        member.preferred_name,
+        member.full_name,
+        member.email,
+        member.phone_number,
+        member.status,
+        member.address1,
+        member.address2,
+        member.city,
+        member.region,
+        member.postal_code,
+        member.number,
+        member.pronouns.join(" "),
+        member.volunteer_interest,
+        *year_values(member)
+      ]
+      stream.write(CSV.generate_line(row))
+    end
+  end
+
+  private
+
+  def year_headers
+    @year_range.flat_map { |year| ["#{year}_amount", "#{year}_started_at"] }
+  end
+
+  def year_values(member)
+    @year_range.flat_map { |year|
+      started_at = member["#{year}_started_at"]
+      amount = member["#{year}_amount"].to_i
+      if started_at
+        [amount / 100, started_at.to_s(:short_date)]
+      elsif amount > 0
+        [amount / 100, nil]
+      else
+        [nil, nil]
+      end
+    }
+  end
+end

app/models/member.rb

@@ -45,9 +45,13 @@ class Member < ApplicationRecord
   scope :active_on, ->(date) { joins(:loan_summaries).merge(LoanSummary.active_on(date)).distinct }
   scope :with_outstanding_items, ->(date) { joins(:loan_summaries).merge(LoanSummary.overdue_as_of(date)).distinct }
   scope :volunteer, -> { where(volunteer_interest: true) }
-
   scope :by_full_name, -> { order(full_name: :desc) }
 
+  scope :for_export, -> {
+    joins("LEFT OUTER JOIN (#{Membership.for_export.to_sql}) AS maby ON members.id = maby.member_id").select("members.*, maby.*")
+      .order("id ASC")
+  }
+
   before_validation :strip_phone_number
   before_validation :set_default_address_fields
   before_validation :downcase_email
@@ -93,6 +97,15 @@ def self.pronoun_list
     PRONOUNS
   end
 
+  def naively_split_name
+    name_parts = member.full_name.split(" ")
+    if name_parts.size == 2
+      [name_parts[0], "", name_parts[1]]
+    else
+      [name_parts[0], name_parts[1], name_parts[2..]&.join(" ")]
+    end
+  end
+
   def display_pronouns
     pronouns.reject(&:empty?).join(", ")
   end

app/models/membership.rb

@@ -10,9 +10,26 @@ class PendingMembership < StandardError; end
   scope :ended, -> { where("ended_at <= ?", Time.current).order("ended_at ASC") }
   scope :expiring_before, ->(date) { where("ended_at <= ?", date) }
 
+  scope :for_export, -> {
+    select_clauses = year_range_for_export.flat_map { |year|
+      [
+        %{SUM(adjustments.amount_cents * -1) FILTER (WHERE date_part('year', adjustments.created_at) = #{year}) AS "#{year}_amount"},
+        %{MAX(memberships.started_at) FILTER (WHERE date_part('year', memberships.started_at) = #{year}) AS "#{year}_started_at"}
+      ]
+    }
+
+    left_joins(:adjustment).select(:member_id, *select_clauses).group(:member_id)
+  }
+
   validate :no_overlapping_dates
   validate :start_after_end
 
+  def self.year_range_for_export
+    first_year = minimum(:started_at).year
+    last_year = maximum(:started_at).year
+    (first_year..last_year)
+  end
+
   def amount
     adjustment ? adjustment.amount * -1 : Money.new(0)
   end

app/views/admin/settings/exports/index.html.erb

@@ -1,10 +1,17 @@
 <%= content_for :header do %>
-  <%= index_header "Item Export" %>
+  <%= index_header "Exports" %>
 <% end %>
 
-<h2>Export all items to CSV</h2>
+<h2>Download data in CSV format</h2>
 
-<%= link_to "Download CSV", admin_settings_exports_path, method: :post, class: "btn btn-lg", data: {turbolinks: false} %>
+<ul>
+  <li>
+    <%= link_to "Items", items_admin_settings_exports_path, method: :post, class: "btn btn-lg", data: {turbolinks: false} %>
+  </li>
+  <li>
+    <%= link_to "Members and memberships", members_admin_settings_exports_path, method: :post, class: "btn btn-lg", data: {turbolinks: false} %>
+  </li>
+</ul>
 
 <div class="columns">
   <div class="column col-6">

app/views/layouts/admin.html.erb

@@ -60,7 +60,7 @@
               <% if current_user.admin? %>
                 <li class="nav-item"><%= link_to "Documents", admin_documents_path %></li>
                 <li class="nav-item"><%= link_to "Email Settings", admin_settings_email_settings_path %></li>
-                <li class="nav-item"><%= link_to "Item Export", admin_settings_exports_path %></li>
+                <li class="nav-item"><%= link_to "Exports", admin_settings_exports_path %></li>
               <% end %>
               <li class="nav-item"><%= link_to "Gift Memberships", admin_gift_memberships_path %></li>
               <% if current_user.admin? %>
@@ -152,7 +152,7 @@
                 <li class="menu-item"><%= link_to "Documents", admin_documents_path %></li>
                 <% if current_user.has_role?(:admin) %>
                   <li class="menu-item"><%= link_to "Email Settings", admin_settings_email_settings_path %></li>
-                  <li class="menu-item"><%= link_to "Item Export", admin_settings_exports_path %></li>
+                  <li class="menu-item"><%= link_to "Exports", admin_settings_exports_path %></li>
                 <% end %>
                 <li class="menu-item"><%= link_to "Gift Memberships", admin_gift_memberships_path %></li>
                 <% if current_user.has_role?(:admin) %>

config/routes.rb

@@ -139,7 +139,10 @@
         get :preview, on: :member
       end
       resources :library_updates
-      resources :exports, only: [:index, :create]
+      resources :exports, only: [:index] do
+        post :items, on: :collection
+        post :members, on: :collection
+      end
     end
 
     resources :holds, only: [:index]

test/application_system_test_case.rb

@@ -1,8 +1,5 @@
 require "test_helper"
 
-# The webdrivers gem doesn't work properly for folks using docker-compose
-require "webdrivers/chromedriver" unless ENV["DOCKER"]
-
 Capybara.default_max_wait_time = 5
 
 # Backported from Rails 6.1
@@ -27,7 +24,7 @@
   Capybara::Selenium::Driver.new(
     app,
     browser: :chrome,
-    capabilities: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
+    options: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
       opts.add_argument "--headless"
       opts.add_argument "--no-sandbox"
       opts.add_argument "--disable-gpu"
@@ -42,7 +39,7 @@
     app,
     browser: :remote,
     url: "http://selenium_chrome:4444/wd/hub",
-    capabilities: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
+    options: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
       opts.add_argument "--headless"
       opts.add_argument "--disable-gpu"
       opts.add_argument "--window-size=1400x1800"
@@ -55,7 +52,7 @@
     app,
     browser: :remote,
     url: "http://selenium_chrome:4444/wd/hub",
-    capabilities: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
+    options: Selenium::WebDriver::Chrome::Options.new.tap do |opts|
       opts.add_argument "--window-size=1400x1800"
     end
   )

test/lib/member_exporter_test.rb

@@ -0,0 +1,15 @@
+require "test_helper"
+
+class MemberExporterTest < ActiveSupport::TestCase
+  test "exports Members" do
+    create(:membership, started_at: Date.new(2020, 3, 4))
+
+    exporter = MemberExporter.new
+    stream = StringIO.new
+    exporter.export(stream)
+
+    stream.rewind
+
+    assert_equal 2, stream.read.lines.size
+  end
+end