Imagine if we could run 1000's of queries nightly in Quepid!!!

Gemfile

@@ -91,3 +91,5 @@ group :test do
 end
 
 gem 'importmap-rails', '~> 2.0'
+
+gem 'ferrum', '~> 0.15'

Gemfile.lock

@@ -167,6 +167,11 @@ GEM
       faraday (>= 1, < 3)
     faraday-net_http (3.1.0)
       net-http
+    ferrum (0.15)
+      addressable (~> 2.5)
+      concurrent-ruby (~> 1.1)
+      webrick (~> 1.7)
+      websocket-driver (~> 0.7)
     ffi (1.16.3)
     font-awesome-sass (6.5.1)
       sassc (~> 2.0)
@@ -507,6 +512,7 @@ DEPENDENCIES
   derailed_benchmarks
   devise (>= 4.6.2)
   devise_invitable (~> 2.0)
+  ferrum (~> 0.15)
   font-awesome-sass
   gabba
   importmap-rails (~> 2.0)

app/assets/javascripts/app_stripped.js

@@ -0,0 +1,13 @@
+'use strict';
+
+angular.module('QuepidApp', [
+  'UtilitiesModule',
+  'ngRoute',
+  'ngCookies',
+  'ngSanitize',
+  'ui.bootstrap',
+  'o19s.splainer-search',
+  'angular-flash.service',
+  'angular-flash.flash-alert-directive',
+  'templates'
+]);

app/assets/javascripts/components/export_case/_modal.html

@@ -29,7 +29,7 @@ <h3 class="modal-title">Export Case: <span class="modal-case">{{ ctrl.theCase.ca
       Detailed export is only supported from the individual Case view.
     </p>
     <span class="help-block">
-      CSV file with <code>Team Name,Case Name,Case ID,Query Text,Doc ID,Title,Rating,Field1,...,FieldN</code> where <code>Field1,...,FieldN</code> are specified under <strong>Settings</strong> in the <strong>Displayed Fields</strong> field.
+      CSV file with <code>Team Name,Case Name,Case ID,Query Text,Doc ID,Position,Title,Rating,Field1,...,FieldN</code> where <code>Field1,...,FieldN</code> are specified under <strong>Settings</strong> in the <strong>Displayed Fields</strong> field.
     </span>
   </div>
 

app/assets/javascripts/controllers/mainCtrl.js

@@ -106,15 +106,25 @@ angular.module('QuepidApp')
                   flash.to('search-error').error = '';
 
                   bootstrapped = true;
-                  return queriesSvc.searchAll()
-                    .then(function() {
-                      flash.success = 'All queries finished successfully!';
-                    }, function(errorMsg) {
-                      var mainErrorMsg = 'Some queries failed to resolve!';
-
-                      flash.error = mainErrorMsg;
-                      flash.to('search-error').error = errorMsg;
-                    });
+
+                  console.log(queriesSvc.queryArray().length);
+                  if (queriesSvc.queryArray().length <= 29) {
+                    console.log('About to call queriesSvc.searchAll');
+                    return queriesSvc.searchAll()
+                      .then(function() {
+                        flash.success = 'All queries finished successfully!';
+                      }, function(errorMsg) {
+                        var mainErrorMsg = 'Some queries failed to resolve!';
+
+                        flash.error = mainErrorMsg;
+                        flash.to('search-error').error = errorMsg;
+                      });
+                  }
+                  else {
+                    queriesSvc.requireManualTrigger = true;
+                    flash.success = 'You have ' + queriesSvc.queryArray().length + ' queries in this case and must manually trigger running this!';
+                    return;
+                  }
                 }
               });
           });
@@ -143,7 +153,6 @@ angular.module('QuepidApp')
       }
       else if ( caseNo > 0 ) {
         queriesSvc.querySearchPromiseReset();
-
         bootstrapCase()
           .then(function() {
             loadQueries();

app/assets/javascripts/controllers/queriesStrippedCtrl.js

@@ -0,0 +1,60 @@
+'use strict';
+/*jslint latedef:false*/
+
+angular.module('QuepidApp')
+  .controller('QueriesStrippedCtrl', [
+    '$scope',    
+    'queriesSvc',    
+    'querySnapshotSvc',
+    'caseSvc',
+    function (
+      $scope,     
+      queriesSvc,   
+      querySnapshotSvc,
+      caseSvc
+    ) {
+      $scope.queriesSvc = queriesSvc;
+      $scope.caseSvc = caseSvc;
+
+      $scope.queries                  = {};
+
+      $scope.queries.queriesChanged = function() {
+        return queriesSvc.version();
+      };
+
+
+      // get all the queries for this case for the query service
+      $scope.queriesList = [];
+      $scope.$watch(function(){
+        // only call if the query service has new information!
+        return queriesSvc.version();
+      }, function(){
+        $scope.queriesList = queriesSvc.queryArray();
+        updateBatchInfo();
+      });
+
+      $scope.snapshotPayload = function() {
+        if (!$scope.searching()){
+          return querySnapshotSvc.createSnapshotPayload('', true, false, queriesSvc.queryArray());
+        }
+      };
+
+      $scope.searching = function() {
+        return queriesSvc.hasUnscoredQueries();
+      };
+      $scope.batchPosition = 0;
+      $scope.batchSize = 0;
+      function getBatchPosition() {
+        return queriesSvc.scoredQueryCount();
+      }
+
+      function updateBatchInfo() {
+        $scope.batchSize = queriesSvc.queryCount();
+        $scope.batchPosition = queriesSvc.scoredQueryCount();
+      }
+      $scope.$watch(getBatchPosition, updateBatchInfo);
+
+
+
+    }
+  ]);

app/assets/javascripts/controllers/queryParams.js

@@ -102,6 +102,7 @@ angular.module('QuepidApp')
           args:           $scope.settings.selectedTry.args,
           curator_vars:    $scope.settings.selectedTry.curatorVarsDict(),
           escape_query:   $scope.settings.selectedTry.escapeQuery,
+          background_queries:   $scope.settings.selectedTry.backgroundQueries,
           api_method:     $scope.settings.selectedTry.apiMethod,
           custom_headers: $scope.settings.selectedTry.customHeaders,
           field_spec:     $scope.settings.selectedTry.fieldSpec,

app/assets/javascripts/controllers/strippedCtrl.js

@@ -0,0 +1,160 @@
+'use strict';
+
+angular.module('QuepidApp')
+  // there's a lot of dependencies here, but this guy
+  // is responsible for bootstrapping everyone so...
+  .controller('StrippedCtrl', [
+    '$scope', '$routeParams', '$log',
+    'flash',
+    'caseSvc', 'settingsSvc', 'caseTryNavSvc',
+    'queryViewSvc', 'queriesSvc', 'docCacheSvc',
+    function (
+      $scope, $routeParams, $log,
+      flash,
+      caseSvc, settingsSvc, caseTryNavSvc,
+      queryViewSvc, queriesSvc, docCacheSvc
+    ) {
+      $log.debug('NEW STRIPPED MAIN CTRLr');
+
+      var caseNo  = parseInt($routeParams.caseNo, 10);
+      var tryNo   = parseInt($routeParams.tryNo, 10);
+      var queryNo   = parseInt($routeParams.queryNo, 10);
+
+      var initialCaseNo = angular.copy(caseTryNavSvc.getCaseNo());
+
+      var caseChanged = function() {
+        return initialCaseNo !== caseNo;
+      };
+
+      var getSearchEngine = function(tryNo) {
+        var settings = settingsSvc.editableSettings();
+        if (settings.hasOwnProperty('getTry')) {
+          var aTry = settings.getTry(tryNo);
+          if (aTry) {
+            return aTry.searchUrl;
+          }
+        }
+        return null;
+      };
+
+      var searchEngineChanged = function() {
+        return getSearchEngine(caseTryNavSvc.getTryNo()) !== getSearchEngine(tryNo);
+      };
+
+      var init = function() {
+        // Make sure we empty stuff from the previous case
+        if ( caseChanged() ) {
+          queriesSvc.reset();
+        }
+
+        angular.forEach(queriesSvc.queries, function(query) {
+          query.reset();
+        });
+      };
+
+      var bootstrapCase = function() {
+        return caseSvc.get(caseNo)
+          .then(function(acase) {
+            if (angular.isUndefined(acase)){
+              throw new Error('Could not retrieve case ' + caseNo + '.   Confirm that the case has been shared with you via a team you are a member of!');
+            }
+
+            caseSvc.selectTheCase(acase);
+            settingsSvc.setCaseTries(acase.tries);
+            if ( isNaN(tryNo) ) {  // If we didn't specify a tryNo via the URL
+              tryNo = acase.lastTry;
+            }
+
+            settingsSvc.setCurrentTry(tryNo);
+            if (!settingsSvc.isTrySelected()){
+              flash.to('search-error').error = 'The try that was specified for the case does not actually exist!';
+            }
+            else {
+              if (settingsSvc.editableSettings().proxyRequests === true){
+                $scope.showTLSChangeWarning = false;
+              }
+              else if  (caseTryNavSvc.needToRedirectQuepidProtocol(settingsSvc.editableSettings().searchUrl)){
+                $log.info('Need to redirect browser to different TLS');
+                throw new Error('Need to change to different TLS'); // Signal that we need to change TLS.              
+              }
+            }
+          });
+      };
+
+      var loadQueries = function() {
+        var newSettings = settingsSvc.editableSettings();
+        if ( caseChanged() || searchEngineChanged() ) {
+          if ( caseChanged() ) {
+            queryViewSvc.reset();
+            docCacheSvc.empty();
+          }
+          docCacheSvc.invalidate();
+        }
+
+        return docCacheSvc.update(newSettings)
+          .then(function() {
+            var bootstrapped = false;
+
+            return queriesSvc.changeSettings2(caseNo, queryNo, newSettings)
+              .then(function() {
+                if (!bootstrapped) {
+                  flash.error                    = '';
+                  flash.success                  = '';
+                  flash.to('search-error').error = '';
+
+                  bootstrapped = true;
+
+                  return queriesSvc.searchAll()
+                    .then(function() {
+                      flash.success = 'All queries finished successfully!';
+                    }, function(errorMsg) {
+                      var mainErrorMsg = 'Some queries failed to resolve!';
+
+                      flash.error = mainErrorMsg;
+                      flash.to('search-error').error = errorMsg;
+                    });                  
+                }
+              });
+          });
+      };
+
+      init();
+
+      caseTryNavSvc.navigationCompleted({
+        caseNo:       caseNo,
+        tryNo:        tryNo
+      });
+
+      // While not perfect, at least the site doesn't blow up if you don't
+      // have any cases.
+      if ( caseNo === 0 ) {
+        flash.error = 'You don\'t have any Cases created in Quepid.  Click \'Create a Case\' from the Relevancy Cases dropdown to get started.';
+      }
+      else if ( caseNo > 0 ) {
+        queriesSvc.querySearchPromiseReset();
+        bootstrapCase()
+          .then(function() {
+            loadQueries();
+          }).catch(function(error) {            
+            // brittle logic, but check if we throw the TLS error or if it's from something else.'
+            var message = error.message;
+            if (message === 'Need to change to different TLS'){
+              var resultsTuple = caseTryNavSvc.swapQuepidUrlTLS();
+
+              var quepidUrlToSwitchTo = resultsTuple[0];
+              var protocolToSwitchTo = resultsTuple[1];
+
+              flash.to('search-error').error = '<a href="' + quepidUrlToSwitchTo + '" class="btn btn-primary form-control">Click Here to <span class="glyphicon glyphicon-refresh"></span> Reload Quepid in <code>' + protocolToSwitchTo + '</code> Protocol!';
+            }
+            else if (message.startsWith('Could not retrieve case')){
+              flash.to('search-error').error = message;
+            }
+            else {
+              flash.to('search-error').error = 'Could not load the case ' + caseNo + ' due to: ' + message;
+            }
+          });
+
+
+      }
+    }
+  ]);

app/assets/javascripts/core.js

@@ -78,7 +78,6 @@
 //= require_tree ./services
 //= require_tree ./values
 //= require_tree ../templates
-//= require_tree ./components
 //= require footer
 //= require tether-shepherd/dist/js/tether
 //= require tether-shepherd/dist/js/shepherd

app/assets/javascripts/core_stripped.js

@@ -0,0 +1,42 @@
+// This is a manifest file that'll be compiled into core.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+// This file loads all the below JavaScript files and can be very slow in the dev mode.
+//
+// This file is meant for the stripped down Angular Core app used by AgentQ.
+//
+//= require jquery
+
+
+//= require angular/angular
+
+//= require angular-resource/angular-resource
+//= require angular-cookies/angular-cookies
+//= require angular-route/angular-route
+//= require angular-sanitize/angular-sanitize
+//= require angular-ui-bootstrap/dist/ui-bootstrap
+//= require angular-rails-templates
+//= require splainer-search/splainer-search
+//= require angular-flash/dist/angular-flash
+//= require urijs/src/URI
+//= require utilitiesModule
+//= require app_stripped
+//= require routes
+//= require_tree ./components
+//= require_tree ./controllers
+//= require_tree ./directives
+//= require_tree ./factories
+//= require_tree ./filters
+//= require_tree ./services
+//= require_tree ./values
+//= require_tree ../templates
+//= require_tree ./components

app/assets/javascripts/directives/queriesStripped.js

@@ -0,0 +1,14 @@
+'use strict';
+
+angular.module('QuepidApp')
+  .directive('queriesStripped', [
+    function () {
+      return {
+        restrict: 'E',
+        transclude: true,
+        controller: 'QueriesStrippedCtrl',
+        templateUrl: 'views/queries-stripped.html',
+        replace: true
+      };
+    }
+  ]);

app/assets/javascripts/factories/TryFactory.js

@@ -40,6 +40,7 @@
       self.args          = data.args;
       self.deleted       = false;
       self.escapeQuery   = data.escape_query;
+      self.backgroundQueries = data.background_queries;
       self.apiMethod     = data.api_method;
       self.customHeaders = data.custom_headers;
       self.fieldSpec     = data.field_spec;