diff --git a/Gemfile b/Gemfile index 28b89130d..0c1968bdf 100755 --- a/Gemfile +++ b/Gemfile @@ -91,3 +91,5 @@ group :test do end gem 'importmap-rails', '~> 2.0' + +gem 'ferrum', '~> 0.15' diff --git a/Gemfile.lock b/Gemfile.lock index bf0909f87..a0728f6b7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -167,6 +167,11 @@ GEM faraday (>= 1, < 3) faraday-net_http (3.1.0) net-http + ferrum (0.15) + addressable (~> 2.5) + concurrent-ruby (~> 1.1) + webrick (~> 1.7) + websocket-driver (~> 0.7) ffi (1.16.3) font-awesome-sass (6.5.1) sassc (~> 2.0) @@ -507,6 +512,7 @@ DEPENDENCIES derailed_benchmarks devise (>= 4.6.2) devise_invitable (~> 2.0) + ferrum (~> 0.15) font-awesome-sass gabba importmap-rails (~> 2.0) diff --git a/app/assets/javascripts/app_stripped.js b/app/assets/javascripts/app_stripped.js new file mode 100644 index 000000000..07979c1fb --- /dev/null +++ b/app/assets/javascripts/app_stripped.js @@ -0,0 +1,13 @@ +'use strict'; + +angular.module('QuepidApp', [ + 'UtilitiesModule', + 'ngRoute', + 'ngCookies', + 'ngSanitize', + 'ui.bootstrap', + 'o19s.splainer-search', + 'angular-flash.service', + 'angular-flash.flash-alert-directive', + 'templates' +]); diff --git a/app/assets/javascripts/components/export_case/_modal.html b/app/assets/javascripts/components/export_case/_modal.html index 87c50f1d6..48ab1964a 100644 --- a/app/assets/javascripts/components/export_case/_modal.html +++ b/app/assets/javascripts/components/export_case/_modal.html @@ -29,7 +29,7 @@

Export Case: {{ ctrl.theCase.ca Detailed export is only supported from the individual Case view.

- CSV file with Team Name,Case Name,Case ID,Query Text,Doc ID,Title,Rating,Field1,...,FieldN where Field1,...,FieldN are specified under Settings in the Displayed Fields field. + CSV file with Team Name,Case Name,Case ID,Query Text,Doc ID,Position,Title,Rating,Field1,...,FieldN where Field1,...,FieldN are specified under Settings in the Displayed Fields field. diff --git a/app/assets/javascripts/controllers/mainCtrl.js b/app/assets/javascripts/controllers/mainCtrl.js index 85c16c2ca..b28b09918 100644 --- a/app/assets/javascripts/controllers/mainCtrl.js +++ b/app/assets/javascripts/controllers/mainCtrl.js @@ -106,15 +106,25 @@ angular.module('QuepidApp') flash.to('search-error').error = ''; bootstrapped = true; - return queriesSvc.searchAll() - .then(function() { - flash.success = 'All queries finished successfully!'; - }, function(errorMsg) { - var mainErrorMsg = 'Some queries failed to resolve!'; - - flash.error = mainErrorMsg; - flash.to('search-error').error = errorMsg; - }); + + console.log(queriesSvc.queryArray().length); + if (queriesSvc.queryArray().length <= 29) { + console.log('About to call queriesSvc.searchAll'); + return queriesSvc.searchAll() + .then(function() { + flash.success = 'All queries finished successfully!'; + }, function(errorMsg) { + var mainErrorMsg = 'Some queries failed to resolve!'; + + flash.error = mainErrorMsg; + flash.to('search-error').error = errorMsg; + }); + } + else { + queriesSvc.requireManualTrigger = true; + flash.success = 'You have ' + queriesSvc.queryArray().length + ' queries in this case and must manually trigger running this!'; + return; + } } }); }); @@ -143,7 +153,6 @@ angular.module('QuepidApp') } else if ( caseNo > 0 ) { queriesSvc.querySearchPromiseReset(); - bootstrapCase() .then(function() { loadQueries(); diff --git a/app/assets/javascripts/controllers/queriesStrippedCtrl.js b/app/assets/javascripts/controllers/queriesStrippedCtrl.js new file mode 100644 index 000000000..4bba208a6 --- /dev/null +++ b/app/assets/javascripts/controllers/queriesStrippedCtrl.js @@ -0,0 +1,60 @@ +'use strict'; +/*jslint latedef:false*/ + +angular.module('QuepidApp') + .controller('QueriesStrippedCtrl', [ + '$scope', + 'queriesSvc', + 'querySnapshotSvc', + 'caseSvc', + function ( + $scope, + queriesSvc, + querySnapshotSvc, + caseSvc + ) { + $scope.queriesSvc = queriesSvc; + $scope.caseSvc = caseSvc; + + $scope.queries = {}; + + $scope.queries.queriesChanged = function() { + return queriesSvc.version(); + }; + + + // get all the queries for this case for the query service + $scope.queriesList = []; + $scope.$watch(function(){ + // only call if the query service has new information! + return queriesSvc.version(); + }, function(){ + $scope.queriesList = queriesSvc.queryArray(); + updateBatchInfo(); + }); + + $scope.snapshotPayload = function() { + if (!$scope.searching()){ + return querySnapshotSvc.createSnapshotPayload('', true, false, queriesSvc.queryArray()); + } + }; + + $scope.searching = function() { + return queriesSvc.hasUnscoredQueries(); + }; + $scope.batchPosition = 0; + $scope.batchSize = 0; + function getBatchPosition() { + return queriesSvc.scoredQueryCount(); + } + + function updateBatchInfo() { + $scope.batchSize = queriesSvc.queryCount(); + $scope.batchPosition = queriesSvc.scoredQueryCount(); + } + $scope.$watch(getBatchPosition, updateBatchInfo); + + + + } + ]); diff --git a/app/assets/javascripts/controllers/queryParams.js b/app/assets/javascripts/controllers/queryParams.js index 0764f3c51..c207ea17f 100644 --- a/app/assets/javascripts/controllers/queryParams.js +++ b/app/assets/javascripts/controllers/queryParams.js @@ -102,6 +102,7 @@ angular.module('QuepidApp') args: $scope.settings.selectedTry.args, curator_vars: $scope.settings.selectedTry.curatorVarsDict(), escape_query: $scope.settings.selectedTry.escapeQuery, + background_queries: $scope.settings.selectedTry.backgroundQueries, api_method: $scope.settings.selectedTry.apiMethod, custom_headers: $scope.settings.selectedTry.customHeaders, field_spec: $scope.settings.selectedTry.fieldSpec, diff --git a/app/assets/javascripts/controllers/strippedCtrl.js b/app/assets/javascripts/controllers/strippedCtrl.js new file mode 100644 index 000000000..0d125e386 --- /dev/null +++ b/app/assets/javascripts/controllers/strippedCtrl.js @@ -0,0 +1,160 @@ +'use strict'; + +angular.module('QuepidApp') + // there's a lot of dependencies here, but this guy + // is responsible for bootstrapping everyone so... + .controller('StrippedCtrl', [ + '$scope', '$routeParams', '$log', + 'flash', + 'caseSvc', 'settingsSvc', 'caseTryNavSvc', + 'queryViewSvc', 'queriesSvc', 'docCacheSvc', + function ( + $scope, $routeParams, $log, + flash, + caseSvc, settingsSvc, caseTryNavSvc, + queryViewSvc, queriesSvc, docCacheSvc + ) { + $log.debug('NEW STRIPPED MAIN CTRLr'); + + var caseNo = parseInt($routeParams.caseNo, 10); + var tryNo = parseInt($routeParams.tryNo, 10); + var queryNo = parseInt($routeParams.queryNo, 10); + + var initialCaseNo = angular.copy(caseTryNavSvc.getCaseNo()); + + var caseChanged = function() { + return initialCaseNo !== caseNo; + }; + + var getSearchEngine = function(tryNo) { + var settings = settingsSvc.editableSettings(); + if (settings.hasOwnProperty('getTry')) { + var aTry = settings.getTry(tryNo); + if (aTry) { + return aTry.searchUrl; + } + } + return null; + }; + + var searchEngineChanged = function() { + return getSearchEngine(caseTryNavSvc.getTryNo()) !== getSearchEngine(tryNo); + }; + + var init = function() { + // Make sure we empty stuff from the previous case + if ( caseChanged() ) { + queriesSvc.reset(); + } + + angular.forEach(queriesSvc.queries, function(query) { + query.reset(); + }); + }; + + var bootstrapCase = function() { + return caseSvc.get(caseNo) + .then(function(acase) { + if (angular.isUndefined(acase)){ + throw new Error('Could not retrieve case ' + caseNo + '. Confirm that the case has been shared with you via a team you are a member of!'); + } + + caseSvc.selectTheCase(acase); + settingsSvc.setCaseTries(acase.tries); + if ( isNaN(tryNo) ) { // If we didn't specify a tryNo via the URL + tryNo = acase.lastTry; + } + + settingsSvc.setCurrentTry(tryNo); + if (!settingsSvc.isTrySelected()){ + flash.to('search-error').error = 'The try that was specified for the case does not actually exist!'; + } + else { + if (settingsSvc.editableSettings().proxyRequests === true){ + $scope.showTLSChangeWarning = false; + } + else if (caseTryNavSvc.needToRedirectQuepidProtocol(settingsSvc.editableSettings().searchUrl)){ + $log.info('Need to redirect browser to different TLS'); + throw new Error('Need to change to different TLS'); // Signal that we need to change TLS. + } + } + }); + }; + + var loadQueries = function() { + var newSettings = settingsSvc.editableSettings(); + if ( caseChanged() || searchEngineChanged() ) { + if ( caseChanged() ) { + queryViewSvc.reset(); + docCacheSvc.empty(); + } + docCacheSvc.invalidate(); + } + + return docCacheSvc.update(newSettings) + .then(function() { + var bootstrapped = false; + + return queriesSvc.changeSettings2(caseNo, queryNo, newSettings) + .then(function() { + if (!bootstrapped) { + flash.error = ''; + flash.success = ''; + flash.to('search-error').error = ''; + + bootstrapped = true; + + return queriesSvc.searchAll() + .then(function() { + flash.success = 'All queries finished successfully!'; + }, function(errorMsg) { + var mainErrorMsg = 'Some queries failed to resolve!'; + + flash.error = mainErrorMsg; + flash.to('search-error').error = errorMsg; + }); + } + }); + }); + }; + + init(); + + caseTryNavSvc.navigationCompleted({ + caseNo: caseNo, + tryNo: tryNo + }); + + // While not perfect, at least the site doesn't blow up if you don't + // have any cases. + if ( caseNo === 0 ) { + flash.error = 'You don\'t have any Cases created in Quepid. Click \'Create a Case\' from the Relevancy Cases dropdown to get started.'; + } + else if ( caseNo > 0 ) { + queriesSvc.querySearchPromiseReset(); + bootstrapCase() + .then(function() { + loadQueries(); + }).catch(function(error) { + // brittle logic, but check if we throw the TLS error or if it's from something else.' + var message = error.message; + if (message === 'Need to change to different TLS'){ + var resultsTuple = caseTryNavSvc.swapQuepidUrlTLS(); + + var quepidUrlToSwitchTo = resultsTuple[0]; + var protocolToSwitchTo = resultsTuple[1]; + + flash.to('search-error').error = 'Click Here to Reload Quepid in ' + protocolToSwitchTo + ' Protocol!'; + } + else if (message.startsWith('Could not retrieve case')){ + flash.to('search-error').error = message; + } + else { + flash.to('search-error').error = 'Could not load the case ' + caseNo + ' due to: ' + message; + } + }); + + + } + } + ]); diff --git a/app/assets/javascripts/core.js b/app/assets/javascripts/core.js index cc7ebb001..d9c482b04 100644 --- a/app/assets/javascripts/core.js +++ b/app/assets/javascripts/core.js @@ -78,7 +78,6 @@ //= require_tree ./services //= require_tree ./values //= require_tree ../templates -//= require_tree ./components //= require footer //= require tether-shepherd/dist/js/tether //= require tether-shepherd/dist/js/shepherd diff --git a/app/assets/javascripts/core_stripped.js b/app/assets/javascripts/core_stripped.js new file mode 100644 index 000000000..f06f448f6 --- /dev/null +++ b/app/assets/javascripts/core_stripped.js @@ -0,0 +1,42 @@ +// This is a manifest file that'll be compiled into core.js, which will include all the files +// listed below. +// +// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts, +// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path. +// +// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the +// compiled file. +// +// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details +// about supported directives. +// +// This file loads all the below JavaScript files and can be very slow in the dev mode. +// +// This file is meant for the stripped down Angular Core app used by AgentQ. +// +//= require jquery + + +//= require angular/angular + +//= require angular-resource/angular-resource +//= require angular-cookies/angular-cookies +//= require angular-route/angular-route +//= require angular-sanitize/angular-sanitize +//= require angular-ui-bootstrap/dist/ui-bootstrap +//= require angular-rails-templates +//= require splainer-search/splainer-search +//= require angular-flash/dist/angular-flash +//= require urijs/src/URI +//= require utilitiesModule +//= require app_stripped +//= require routes +//= require_tree ./components +//= require_tree ./controllers +//= require_tree ./directives +//= require_tree ./factories +//= require_tree ./filters +//= require_tree ./services +//= require_tree ./values +//= require_tree ../templates +//= require_tree ./components diff --git a/app/assets/javascripts/directives/queriesStripped.js b/app/assets/javascripts/directives/queriesStripped.js new file mode 100644 index 000000000..26cf1e02d --- /dev/null +++ b/app/assets/javascripts/directives/queriesStripped.js @@ -0,0 +1,14 @@ +'use strict'; + +angular.module('QuepidApp') + .directive('queriesStripped', [ + function () { + return { + restrict: 'E', + transclude: true, + controller: 'QueriesStrippedCtrl', + templateUrl: 'views/queries-stripped.html', + replace: true + }; + } + ]); diff --git a/app/assets/javascripts/factories/TryFactory.js b/app/assets/javascripts/factories/TryFactory.js index f764d0bf6..48de322bf 100644 --- a/app/assets/javascripts/factories/TryFactory.js +++ b/app/assets/javascripts/factories/TryFactory.js @@ -40,6 +40,7 @@ self.args = data.args; self.deleted = false; self.escapeQuery = data.escape_query; + self.backgroundQueries = data.background_queries; self.apiMethod = data.api_method; self.customHeaders = data.custom_headers; self.fieldSpec = data.field_spec; diff --git a/app/assets/javascripts/routes.js b/app/assets/javascripts/routes.js index 974b61e84..5352189e4 100644 --- a/app/assets/javascripts/routes.js +++ b/app/assets/javascripts/routes.js @@ -33,6 +33,11 @@ angular.module('QuepidApp') controller: 'MainCtrl', reloadOnSearch: false }) + .when('/case/:caseNo/query/:queryNo', { + templateUrl: 'views/queriesStripped.html', + controller: 'StrippedCtrl', + reloadOnSearch: false + }) .when('/cases', { templateUrl: 'views/cases/index.html', controller: 'CasesCtrl' diff --git a/app/assets/javascripts/services/caseCSVSvc.js b/app/assets/javascripts/services/caseCSVSvc.js index 871281b29..23f16e74e 100644 --- a/app/assets/javascripts/services/caseCSVSvc.js +++ b/app/assets/javascripts/services/caseCSVSvc.js @@ -79,6 +79,7 @@ 'Case ID', 'Query Text', 'Doc ID', + 'Doc Position', 'Title', 'Rating', ]; @@ -353,7 +354,7 @@ csvContent += dataString + EOL; } else { - angular.forEach(docs, function (doc) { + angular.forEach(docs, function (doc, index) { var dataString; var infoArray = []; @@ -362,6 +363,7 @@ infoArray.push(stringifyField(aCase.lastScore.case_id)); infoArray.push(stringifyField(query.queryText)); infoArray.push(stringifyField(doc.id)); + infoArray.push(stringifyField(index+1)); infoArray.push(stringifyField(doc.title)); infoArray.push(stringifyField(doc.getRating())); diff --git a/app/assets/javascripts/services/queriesSvc.js b/app/assets/javascripts/services/queriesSvc.js index f279f1ba6..527e57426 100644 --- a/app/assets/javascripts/services/queriesSvc.js +++ b/app/assets/javascripts/services/queriesSvc.js @@ -100,6 +100,10 @@ angular.module('QuepidApp') // custom parser... passedInSettings.searchEngine = 'solr'; } + + if (passedInSettings.backgroundQueries) { + // Similar to how 'static' works, if we have + } else if (passedInSettings.searchEngine === 'searchapi'){ /*jshint evil:true */ eval(passedInSettings.mapperCode); @@ -736,7 +740,7 @@ angular.module('QuepidApp') let querySearchableDeferred = $q.defer(); function bootstrapQueries(caseNo) { querySearchableDeferred = $q.defer(); - var path = 'api/cases/' + caseNo + '/queries?bootstrap=true'; + var path = 'api/cases/' + caseNo + '/queries'; $http.get(path) .then(function(response) { @@ -754,6 +758,33 @@ angular.module('QuepidApp') return querySearchableDeferred.promise; } + + function bootstrapQuery(caseNo, queryNo) { + querySearchableDeferred = $q.defer(); + var path = 'api/cases/' + caseNo + '/queries/' + queryNo; + + $http.get(path) + .then(function(response) { + that.queries = {}; + // wrap the single result in an hash with an array. + let results = { + display_order:[response.data.query_id], + queries: [response.data] + }; + + addQueriesFromResp(results); + + querySearchableDeferred.resolve(); + }, function(response) { + $log.debug('Failed to bootstrap queries: ', response); + return response; + }).catch(function(response) { + $log.debug('Failed to bootstrap queries'); + return response; + }); + + return querySearchableDeferred.promise; + } this.querySearchReady = function() { $log.debug('PROMISE subscribed...'); @@ -784,6 +815,26 @@ angular.module('QuepidApp') caseNo = newCaseNo; return querySearchableDeferred.promise; }; + + this.changeSettings2 = function(newCaseNo, queryNo, newSettings) { + currSettings = newSettings; + + if (caseNo !== newCaseNo) { + scorerSvc.bootstrap(newCaseNo); + bootstrapQuery(newCaseNo, queryNo); + } else { + angular.forEach(this.queries, function(query) { + // TODO update settings for diffs + if (query.diff !== null) { + query.diff.fetch(); + } + }); + querySearchableDeferred.resolve(); + } + + caseNo = newCaseNo; + return querySearchableDeferred.promise; + }; this.searchAll = function() { diff --git a/app/assets/javascripts/services/querySnapshotSvc.js b/app/assets/javascripts/services/querySnapshotSvc.js index 15c58477e..9c9e93de1 100644 --- a/app/assets/javascripts/services/querySnapshotSvc.js +++ b/app/assets/javascripts/services/querySnapshotSvc.js @@ -108,7 +108,8 @@ angular.module('QuepidApp') }); }; - this.addSnapshot = function(name, recordDocumentFields, queries) { + + this.createSnapshotPayload = function(name, recordDocumentFields, recordExplain, queries) { // we may want to refactor the payload structure in the future. var docs = {}; var queriesPayload = {}; @@ -129,8 +130,12 @@ angular.module('QuepidApp') // Save all matches angular.forEach(query.docs, function(doc) { - - var docPayload = {'id': doc.id, 'explain': doc.explain().rawStr(), 'rated_only': false}; + let explain = null; + if (recordExplain) { + explain = doc.explain().rawStr(); + } + var docPayload = {'id': doc.id, 'explain': explain, 'rated_only': false}; + if (recordDocumentFields) { var fields = {}; angular.forEach(Object.values(doc.subsList), function(field) { @@ -161,15 +166,22 @@ angular.module('QuepidApp') }); }); - var saved = { + var payload = { 'snapshot': { 'name': name, 'docs': docs, 'queries': queriesPayload } }; + + return payload; + }; + + this.addSnapshot = function(name, recordDocumentFields, queries) { + + let payload = this.createSnapshotPayload(name, recordDocumentFields, true, queries); - return $http.post('api/cases/' + caseNo + '/snapshots', saved) + return $http.post('api/cases/' + caseNo + '/snapshots', payload) .then(function(response) { return addSnapshotResp([response.data]) .then(function() { diff --git a/app/assets/templates/views/devQueryParams.html b/app/assets/templates/views/devQueryParams.html index aa434cb06..7d84fce32 100644 --- a/app/assets/templates/views/devQueryParams.html +++ b/app/assets/templates/views/devQueryParams.html @@ -252,7 +252,35 @@

Tuning Knobs

- + + + +
+
+ Run Queries in Background +
+ +
+ + +

+ + + + + +

+
+
+ + + diff --git a/app/assets/templates/views/queries-stripped.html b/app/assets/templates/views/queries-stripped.html new file mode 100644 index 000000000..90d570fa9 --- /dev/null +++ b/app/assets/templates/views/queries-stripped.html @@ -0,0 +1,22 @@ +
+
+
+ +
+ Number of Queries: {{ queriesList.length }} +
+ +
+ + + + + + +
+
+
diff --git a/app/assets/templates/views/queriesStripped.html b/app/assets/templates/views/queriesStripped.html new file mode 100644 index 000000000..fa1e42b2e --- /dev/null +++ b/app/assets/templates/views/queriesStripped.html @@ -0,0 +1,28 @@ +
+
+
+
+
+

+ STRIPPED Current case +
+ + {{ caseModel.selectedCase().caseName }} + + + — + + {{ currentTry.selectedTry().formattedName() }} + + + +

+
+
+
+ + + + +
+
diff --git a/app/controllers/api/v1/books/populate_controller.rb b/app/controllers/api/v1/books/populate_controller.rb index 2c9f00205..204eebb63 100644 --- a/app/controllers/api/v1/books/populate_controller.rb +++ b/app/controllers/api/v1/books/populate_controller.rb @@ -20,13 +20,13 @@ class PopulateController < Api::ApiController # # rubocop:disable Layout/LineLength def update - puts "[PopulateController] Request Size is #{number_to_human_size(query_doc_pairs_params.to_s.bytesize)}" + # puts "[PopulateController] Request Size is #{number_to_human_size(query_doc_pairs_params.to_s.bytesize)}" serialized_data = Marshal.dump(query_doc_pairs_params) - puts "[PopulateController] the size of the serialized data is #{number_to_human_size(serialized_data.bytesize)}" + # puts "[PopulateController] the size of the serialized data is #{number_to_human_size(serialized_data.bytesize)}" compressed_data = Zlib::Deflate.deflate(serialized_data) - puts "[PopulateController] the size of the compressed data is #{number_to_human_size(compressed_data.bytesize)}" + # puts "[PopulateController] the size of the compressed data is #{number_to_human_size(compressed_data.bytesize)}" @book.populate_file.attach(io: StringIO.new(compressed_data), filename: "book_populate_#{@book.id}.bin.zip", content_type: 'application/zip') PopulateBookJob.perform_later current_user, @book, @case diff --git a/app/controllers/api/v1/cases/jobs_controller.rb b/app/controllers/api/v1/cases/jobs_controller.rb new file mode 100644 index 000000000..5a2cf410f --- /dev/null +++ b/app/controllers/api/v1/cases/jobs_controller.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Api + module V1 + module Cases + class JobsController < Api::ApiController + before_action :set_case + before_action :check_case + def trigger + RunCaseJob2.perform_later @current_user, @case, root_url + render json: { message: 'scheduled' }, status: :ok + end + end + end + end +end diff --git a/app/controllers/api/v1/cases_controller.rb b/app/controllers/api/v1/cases_controller.rb index d8c6bf4e8..aef0782f3 100644 --- a/app/controllers/api/v1/cases_controller.rb +++ b/app/controllers/api/v1/cases_controller.rb @@ -4,7 +4,7 @@ module Api module V1 # rubocop:disable Metrics/ClassLength class CasesController < Api::ApiController - before_action :set_case, only: [ :show, :update, :destroy ] + before_action :set_case, only: [ :update, :destroy ] before_action :case_with_all_the_bells_whistles, only: [ :show ] before_action :check_case, only: [ :show, :update, :destroy ] diff --git a/app/controllers/api/v1/queries/agent_q_controller.rb b/app/controllers/api/v1/queries/agent_q_controller.rb new file mode 100644 index 000000000..c8e45646b --- /dev/null +++ b/app/controllers/api/v1/queries/agent_q_controller.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'ferrum' +require 'benchmark' + +# NOTE: Auto reloading this file appears to not work. If you change the file, quit Rails and restart it. +# I suspect something about how we are talking to Ferrum/Chrome. +# +# Also browser.quit causes Rails to quit too! +# +# rubocop:disable Metrics/MethodLength +module Api + module V1 + module Queries + class AgentQController < Queries::ApplicationController + # skip_before_action :require_login + skip_before_action :verify_authenticity_token, only: [ :fetch ] + before_action :set_case, only: [ :fetch ] + before_action :check_case, only: [ :fetch ] + before_action :set_case_query, only: [ :fetch ] + before_action :check_query, only: [ :fetch ] + @browser = nil + + # rubocop:disable Metrics/AbcSize + def fetch + api_key = nil + if @current_user.api_keys.empty? + api_key = @current_user.api_keys.create! token: SecureRandom.hex + puts 'we made a api' + else + api_key = @current_user.api_keys.first + end + + puts "api_key is #{api_key.token_digest}" + + if @browser.nil? + puts 'creating Ferrum Browser' + # Launch Ferrum browser + @browser = Ferrum::Browser.new({ + process_timeout: 5, headless: true, + window_size: [ 1280, 800 ], + browser_options: { 'no-sandbox': nil }, + timeout: 60, + pending_connection_errors: false + + }) + end + # @browser.headers.set({ 'Authorization' => "Bearer #{api_key.token_digest}" }) + page = @browser.create_page + page.headers.set( + { 'Authorization' => "Bearer #{api_key.token_digest}" } + ) + + counter = 30 + result = Benchmark.measure do + # Perform a GET request + # + # api_case_query_agent_q_path + puts "root url is #{root_url}" + quepid_url = "#{root_url}case/#{@case.id}/query/#{@query.id}" + # quepid_url = api_case_query_agent_q_url @case, @query + puts "looking up #{quepid_url}" + page.go_to(quepid_url) + # page.network.wait_for_idle + + start_time = Time.zone.now + + # Run the loop for a maximum of 5 seconds + # Timeout.timeout(60) do + loop do + break unless page.at_css('.snapshot-payload').nil? + break if counter.zero? + + # Sleep for a short duration before checking the condition again + + counter -= 1 + sleep 1 + end + puts "Condition met after #{Time.zone.now - start_time} seconds." + end + puts result + + node = page.at_css('.snapshot-payload') + + # @@browser.quit + + if node + render json: node.text, status: :ok + else + render json: { agentq_error: 'boom', counter: counter }, status: :internal_server_error + end + end + # rubocop:enable Metrics/AbcSize + end + end + end +end +# rubocop:enable Metrics/MethodLength diff --git a/app/controllers/api/v1/queries_controller.rb b/app/controllers/api/v1/queries_controller.rb index 6b2d695de..052c6ce49 100644 --- a/app/controllers/api/v1/queries_controller.rb +++ b/app/controllers/api/v1/queries_controller.rb @@ -5,8 +5,8 @@ module V1 class QueriesController < Api::ApiController before_action :set_case before_action :check_case - before_action :set_query, only: [ :update, :destroy ] - before_action :check_query, only: [ :update, :destroy ] + before_action :set_query, only: [ :update, :destroy, :show ] + before_action :check_query, only: [ :update, :destroy, :show ] def index @queries = @case.queries.includes([ :ratings ]) @@ -16,6 +16,9 @@ def index respond_with @queries, @display_order end + def show + end + # rubocop:disable Metrics/MethodLength # rubocop:disable Metrics/AbcSize def create @@ -35,7 +38,8 @@ def create if @query.save @query.insert_at(params[:position].to_i) if params[:position] - @case.save + # next line I think deals with positioning + @case.save # Having this line made inserting a new case trigger a bunch of updates which was slow. Analytics::Tracker.track_query_created_event current_user, @query @@ -82,7 +86,7 @@ def destroy # Make sure queries have the right `arranged_next` and `arranged_at` # values after the query has been removed - @case.rearrange_queries + @case.rearrange_queries # super slow with lots of queires @case.save head :no_content diff --git a/app/controllers/concerns/authentication/current_case_manager.rb b/app/controllers/concerns/authentication/current_case_manager.rb index 8698bb6c4..9dc57e1d7 100644 --- a/app/controllers/concerns/authentication/current_case_manager.rb +++ b/app/controllers/concerns/authentication/current_case_manager.rb @@ -81,7 +81,7 @@ def case_with_all_the_bells_whistles .cases_involved_with .where(id: params[:case_id]) .includes(:tries ) - .preload([ queries: [ :ratings ], tries: [ :curator_variables, :search_endpoint ] ]) + .preload([ :queries, { tries: [ :curator_variables, :search_endpoint ] } ]) .order('tries.try_number DESC') .first end diff --git a/app/controllers/core_controller.rb b/app/controllers/core_controller.rb index ff38171fa..997a23c48 100755 --- a/app/controllers/core_controller.rb +++ b/app/controllers/core_controller.rb @@ -6,6 +6,19 @@ class CoreController < ApplicationController before_action :populate_from_params, except: :new skip_before_action :check_for_announcement + # def require_login + # @current_user = User.find(3) + # end + + # Spiking out can we make an API public? + def authenticate_api! + set_case + return true if @case&.public? || current_user + + render json: { reason: 'Unauthorized!' }, + status: :unauthorized + end + def index end diff --git a/app/controllers/core_stripped_controller.rb b/app/controllers/core_stripped_controller.rb new file mode 100755 index 000000000..c6dd36051 --- /dev/null +++ b/app/controllers/core_stripped_controller.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +# This hosts the stripped down version Angular 1 application that runs in the client. +# It is meant to wrap up the business logic used in interacting with a search endpoint +# and abstract that all away from the caller. We create a HTML page that is populated +# with the resulting query/docs. Currently using the "snapshot" JSON format. +# It is meant to be called by agent_q_controller.rb. +# +class CoreStrippedController < ApplicationController + include ApiKeyAuthenticatable + skip_before_action :check_for_announcement + before_action :authenticate_with_api_key! # , only: [:index] + + def require_login + if request.headers['Authorization'] + token = request.headers['Authorization'] + token = token[7..] # chop off 'Bearer ' portion of the header + current_api_key = ApiKey.find_by token_digest: token + + @current_user = current_api_key.user + end + end + + # Spiking out can we make an API public? + def authenticate_api! + set_case + return true if @case&.public? || current_user + + render json: { reason: 'Unauthorized!' }, + status: :unauthorized + end + + def index + end +end diff --git a/app/jobs/run_case_job.rb b/app/jobs/run_case_job.rb new file mode 100644 index 000000000..b7e4e6dc1 --- /dev/null +++ b/app/jobs/run_case_job.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +class RunCaseJob < ApplicationJob + queue_as :single + sidekiq_options retry: 0 + + @browser = nil + + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/PerceivedComplexity + def perform user, kase + api_key = nil + if user.api_keys.empty? + api_key = user.api_keys.create! token: SecureRandom.hex + puts 'we made a api' + else + api_key = user.api_keys.first + end + + puts "api_key is #{api_key.token_digest}" + + retries = 3 + + begin + if @browser.nil? + puts 'creating Ferrum Browser' + # Launch Ferrum browser + + @browser = Ferrum::Browser.new({ + process_timeout: 5, headless: 'new', + window_size: [ 1280, 800 ], + browser_options: { 'no-sandbox': nil }, + timeout: 60, + pending_connection_errors: false, + js_errors: false + + }) + end + + # @browser.headers.set({ 'Authorization' => "Bearer #{api_key.token_digest}" }) + page = @browser.create_page + page.headers.set({ 'Authorization' => "Bearer #{api_key.token_digest}" }) + + kase.queries.each do |query| + result = Benchmark.measure do + # Perform a GET request + # url = "http://localhost:3000/case/#{kase.id}/query/#{query.id}" + url = "http://localhost:3000/case/#{kase.id}/query/#{query.id}" + puts url + page.go_to(url) + page.network.wait_for_idle + end + + puts result + + start_time = Time.zone.now + + counter = 30 + # Run the loop for a maximum of 5 seconds + # Timeout.timeout(60) do + loop do + break unless page.at_css('.snapshot-payload').nil? + break if counter.zero? + + # Sleep for a short duration before checking the condition again + + counter -= 1 + sleep 1 + end + + puts "Condition met after #{Time.zone.now - start_time} seconds." + + node = page.at_css('.snapshot-payload') + + if node.nil? + puts 'We have a nil Node, so no response' + else + response = node.text + puts response + end + end + rescue Ferrum::BrowserError => e + retries -= 1 + if retries.positive? + puts "Ferrum BrowserError occurred: #{e.message}. Retrying..." + retry + else + puts "Ferrum BrowserError occurred: #{e.message}. Maximum retries reached." + # Handle the error or log it + end + rescue Ferrum::StatusError => e + puts "Ferrum StatusError occurred: #{e.message}." + ensure + # @browser.close if @browser # not yet released + puts 'about to reset browser' + @browser.reset + # @browser.quit + # puts "browser quit" + end + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/PerceivedComplexity +end diff --git a/app/jobs/run_case_job2.rb b/app/jobs/run_case_job2.rb new file mode 100644 index 000000000..acc2e5aa5 --- /dev/null +++ b/app/jobs/run_case_job2.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class RunCaseJob2 < ApplicationJob + queue_as :single + sidekiq_options retry: 0 + + # rubocop:disable Metrics/MethodLength + # rubocop:disable Metrics/AbcSize + def perform user, kase, root_url + api_key = nil + if user.api_keys.empty? + api_key = user.api_keys.create! token: SecureRandom.hex + puts 'we made a api' + else + api_key = user.api_keys.first + end + + puts "api_key is #{api_key.token_digest}" + puts "kase has #{kase.queries.size}" + kase.queries.each do |query| + puts "root url is #{root_url}" + # puts 'changing to https://localhost' + # root_url = 'https://localhost/' + url = "#{root_url}api/cases/#{kase.id}/queries/#{query.id}/agent_q" + puts url + # Create a Faraday connection + connection = Faraday.new(url: url) do |faraday| + # Set the request headers + faraday.headers['Authorization'] = "Bearer #{api_key.token_digest}" + faraday.headers['Accept'] = 'application/json' + faraday.adapter Faraday.default_adapter + faraday.ssl.verify = false # Disable SSL certificate validation + end + + # Make the GET request + response = connection.get + puts 'I did a get' + + if response.success? + # Parse the response body as JSON + data = JSON.parse(response.body) + + puts data + else + puts "Didn't get okay response. #{response.status}" + puts "Response.body: #{response.body}" + + end + end + end + # rubocop:enable Metrics/MethodLength + # rubocop:enable Metrics/AbcSize +end diff --git a/app/views/core_stripped/index.html.erb b/app/views/core_stripped/index.html.erb new file mode 100755 index 000000000..38a2c9b3f --- /dev/null +++ b/app/views/core_stripped/index.html.erb @@ -0,0 +1,17 @@ + + +
+
+
+
+ + + Loading... + +
+
+
+
+
+
+
diff --git a/app/views/layouts/core_stripped.html.erb b/app/views/layouts/core_stripped.html.erb new file mode 100755 index 000000000..6f8e10803 --- /dev/null +++ b/app/views/layouts/core_stripped.html.erb @@ -0,0 +1,35 @@ + + + + +Quepid -- Relevant Search Solved + +<%= csrf_meta_tags %> + + + + + + +
+ <%= yield %> +
+ + + <%= javascript_include_tag 'core_stripped' %> + + + + + diff --git a/config/initializers/assets.rb b/config/initializers/assets.rb index 2c20284dc..584d77af2 100755 --- a/config/initializers/assets.rb +++ b/config/initializers/assets.rb @@ -15,8 +15,8 @@ # Precompile additional assets. # application.js, application.css, and all non-JS/CSS in the app/assets # folder are already added. -Rails.application.config.assets.precompile += %w[ core.css core.js admin.css admin_users.css admin.js admin_users.js - analytics.js ] +Rails.application.config.assets.precompile += %w[ core.css core.js core_stripped.js ] +Rails.application.config.assets.precompile += %w[ admin.css admin_users.css admin.js admin_users.js analytics.js ] Rails.application.config.assets.precompile += %w[ application_spec.js ] # CSS from node modules diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 7f4dce654..ccf23722b 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -2,16 +2,20 @@ # Redis and Sidekiq is used only for sending Google Analytics updates, which is # primarily used by the app.quepid.com install. This checks if we need Sidekiq. -if Rails.application.config.google_analytics_enabled +# if Rails.application.config.google_analytics_enabled - sidekiq_config = { url: ENV.fetch('REDIS_URL', nil), - ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE } } +sidekiq_config = { url: ENV.fetch('REDIS_URL', nil), + ssl_params: { verify_mode: OpenSSL::SSL::VERIFY_NONE } } - Sidekiq.configure_server do |config| - config.redis = sidekiq_config +Sidekiq.configure_server do |config| + config.redis = sidekiq_config + config.capsule('unsafe') do |cap| + cap.concurrency = 1 + cap.queues = %w[single] end +end - Sidekiq.configure_client do |config| - config.redis = sidekiq_config - end +Sidekiq.configure_client do |config| + config.redis = sidekiq_config end +# end diff --git a/config/puma.rb b/config/puma.rb index 339ea631a..4b1eec6b7 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -2,7 +2,8 @@ # Setup taken from https://devcenter.heroku.com/articles/deploying-rails-applications-with-the-puma-web-server#adding-puma-to-your-application -#workers Integer(ENV['WEB_CONCURRENCY'] || 2) +# With AgentQ doing a call back to ourselves, I think we need to bump this... +workers Integer(ENV['WEB_CONCURRENCY'] || 2) #threads_count = Integer(ENV['RAILS_MAX_THREADS'] || 5) #threads threads_count, threads_count diff --git a/config/routes.rb b/config/routes.rb index 2a28f21d7..3cbfe9918 100755 --- a/config/routes.rb +++ b/config/routes.rb @@ -123,6 +123,9 @@ get 'test_exception' => 'api#test_exception' scope module: :v1, constraints: ApiConstraint.new(version: 1, default: true) do + # get '/agent_q/cases/:case_id/queries/:id' => 'agent_q#fetch' + # get '/agent_q/trigger' => 'agent_q#trigger' + resources :users, only: [ :index, :show, :update ] do get '/current' => 'current_user#show', on: :collection end @@ -143,6 +146,8 @@ # different scenarios. resources :cases, except: [ :new, :edit ], param: :case_id resources :cases, only: [] do + get '/jobs/trigger' => 'cases/jobs#trigger' + # Case Tries resources :tries, param: :try_number, except: [ :new ] do post '/duplicate' => 'duplicate_tries#create', as: :duplicate_try @@ -152,12 +157,13 @@ resources :scorers, only: [ :index, :update ], controller: :case_scorers # Case Queries - resources :queries, except: [ :new, :edit, :show ] do + resources :queries, except: [ :new, :edit ] do scope module: :queries do resource :notes, only: [ :show, :update ] resource :options, only: [ :show, :update ] resource :position, only: [ :update ] resource :ratings, only: [ :update, :destroy ] # not actually a singular resource, doc_id in json payload + get 'agent_q' => 'agent_q#fetch' end resource :bulk, only: [] do @@ -250,6 +256,7 @@ # Routes handled by angular get '/case/:id(/try/:try_number)' => 'core#index', as: :case_core + get '/case/:id(/query/:queryNo)' => 'core_stripped#index' get '/cases/new' => 'core#new', as: :case_new get '/cases' => 'core#index' get '/case' => 'core#index' diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake index d6860407a..74d72b1bc 100644 --- a/lib/tasks/assets.rake +++ b/lib/tasks/assets.rake @@ -26,6 +26,6 @@ namespace :assets do # Hook into existing assets:precompile task Rake::Task['assets:precompile'].enhance do - Rake::Task['assets:jupyterlite'].invoke if Rails.env.production? + # Rake::Task['assets:jupyterlite'].invoke if Rails.env.production? end end diff --git a/lib/tasks/sample_data.thor b/lib/tasks/sample_data.thor index b3b6814f6..92c82344d 100644 --- a/lib/tasks/sample_data.thor +++ b/lib/tasks/sample_data.thor @@ -227,6 +227,8 @@ class SampleData < Thor osc.members << osc_member_user unless osc.members.include?(osc_member_user) osc.members << realistic_activity_user unless osc.members.include?(realistic_activity_user) osc.cases << tens_of_queries_case unless osc.members.include?(tens_of_queries_case) + osc.search_endpoints << statedecoded_solr_endpoint unless osc.search_endpoints.include?(statedecoded_solr_endpoint) + print_step 'End of seeding teams................' # Books diff --git a/test/integration/tls_flow_test.rb b/test/integration/tls_flow_test.rb index 480b6188a..fbb2ef9d2 100644 --- a/test/integration/tls_flow_test.rb +++ b/test/integration/tls_flow_test.rb @@ -19,6 +19,7 @@ class TlsFlowTest < ActionDispatch::IntegrationTest assert try_https.search_endpoint.endpoint_url.starts_with?('https') # Navigate to a try that is http TLS protocol + puts "HEre is the kase id: #{kase.id}" get case_core_url(id: kase.id, try_number: try_http.try_number) assert_response :ok