From 2ae9a59aa36d93518660dc27b0f15331b249c500 Mon Sep 17 00:00:00 2001 From: sam marshall Date: Fri, 13 Sep 2024 18:18:23 +0100 Subject: [PATCH] MDL-83119 search_solr: Implement check on connectivity, space usage Implements a status check which confirms that the Solr search engine is available. Optionally, the check can also show a warning if the index grows beyond a certain size. As part of this change, a new API was added in search_solr\engine to allow using http_client (Guzzle) instead of raw Curl; this makes it easier to create mock tests in PHPunit for the new functionality. --- .../engine/solr/classes/check/connection.php | 126 ++++ search/engine/solr/classes/engine.php | 209 +++++- search/engine/solr/lang/en/search_solr.php | 8 + search/engine/solr/lib.php | 43 ++ search/engine/solr/settings.php | 8 + search/engine/solr/tests/engine_test.php | 13 + search/engine/solr/tests/mock_engine_test.php | 634 ++++++++++++++++++ 7 files changed, 1037 insertions(+), 4 deletions(-) create mode 100644 search/engine/solr/classes/check/connection.php create mode 100644 search/engine/solr/lib.php create mode 100644 search/engine/solr/tests/mock_engine_test.php diff --git a/search/engine/solr/classes/check/connection.php b/search/engine/solr/classes/check/connection.php new file mode 100644 index 0000000000000..10eca7cdd4ea0 --- /dev/null +++ b/search/engine/solr/classes/check/connection.php @@ -0,0 +1,126 @@ +. + +namespace search_solr\check; + +use core\check\check; +use core\check\result; +use core\output\html_writer; + +/** + * Check that the connection to Solr works. + * + * @package search_solr + * @copyright 2024 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connection extends check { + #[\Override] + public function get_name(): string { + return get_string('pluginname', 'search_solr'); + } + + #[\Override] + public function get_action_link(): ?\action_link { + return new \action_link( + new \moodle_url('/admin/settings.php', ['section' => 'searchsolr']), + get_string('settings')); + } + + + #[\Override] + public function get_result(): result { + global $CFG; + + $result = result::OK; + $resultstr = ''; + $resultdetails = ''; + + try { + // We do not use manager::instance as this will already try to connect to the engine, + // we only want to do the specific get_status call below and nothing else. So use + // search_engine_instance. We know it will be a Solr instance if we got here. + /** @var \search_solr\engine $engine */ + $engine = \core_search\manager::search_engine_instance(); + + // Get engine status. + $status = $engine->get_status(5); + + $time = number_format($status['time'], 2) . 's'; + $resultstr = get_string('check_time', 'search_solr', $time); + } catch (\Throwable $t) { + $status = [ + 'connected' => false, + 'foundcore' => false, + 'error' => 'Exception when creating search manager: ' . $t->getMessage(), + 'exception' => $t, + ]; + } + + if (!$status['connected']) { + // No connection at all. + $result = result::ERROR; + $resultstr = get_string('check_notconnected', 'search_solr'); + $resultdetails .= \html_writer::tag('p', s($status['error'])); + + } else if (!$status['foundcore']) { + // There's a connection, but the core doesn't seem to exist. + $result = result::ERROR; + $resultstr = get_string('check_nocore', 'search_solr'); + $resultdetails .= \html_writer::tag('p', s($status['error'])); + + } else { + // Errors related to finding the core size only show if the size warning is configured. + $sizelimit = get_config('search_solr', 'indexsizelimit'); + if (!array_key_exists('indexsize', $status)) { + if ($sizelimit) { + $result = result::ERROR; + $resultstr = get_string('check_nosize', 'search_solr'); + $resultdetails .= \html_writer::tag('p', s($status['error'])); + } + } else { + // Show the index size in result, even if we aren't checking it. + $sizestr = get_string( + 'indexsize', + 'search_solr', + display_size($status['indexsize']), + ); + $resultdetails .= \html_writer::tag('p', $sizestr); + if ($sizelimit) { + // Error at specified index size, warning at 90% of it. + $sizewarning = ($sizelimit * 9) / 10; + if ($status['indexsize'] > $sizewarning) { + if ($status['indexsize'] > $sizelimit) { + $resultstr = get_string('check_indextoobig', 'search_solr'); + $result = result::ERROR; + } else { + // We don't say it's too big because it isn't yet, just show the size. + $resultstr = $sizestr; + $result = result::WARNING; + } + } + } + } + } + + $ex = $status['exception'] ?? null; + if ($ex) { + $resultdetails .= \html_writer::tag('pre', str_replace($CFG->dirroot, '', s($ex->getTraceAsString()))); + } + + return new result($result, $resultstr, $resultdetails); + } +} diff --git a/search/engine/solr/classes/engine.php b/search/engine/solr/classes/engine.php index 073a64b2648d0..4428b976cb0b4 100644 --- a/search/engine/solr/classes/engine.php +++ b/search/engine/solr/classes/engine.php @@ -1340,6 +1340,102 @@ public function is_installed() { return function_exists('solr_get_version'); } + /** @var int When using the capath option, we generate a bundle containing all the pem files, cached 10 mins. */ + const CA_PATH_CACHE_TIME = 600; + + /** @var int Expired cache files are deleted after this many seconds. */ + const CA_PATH_CACHE_DELETE_AFTER = 60; + + /** + * Gets status of Solr server. + * + * The result has the following fields: + * - connected - true if we got a valid JSON response from server + * - foundcore - true if we found the core defined in config (this could be false if schema not set up) + * + * It may have these other fields: + * - error - text if anything went wrong + * - exception - if an exception was thrown + * - indexsize - index size in bytes if we found what it is + * + * @param int $timeout Optional timeout in seconds, otherwise uses config value + * @return array Array with information about status + * @since Moodle 5.0 + */ + public function get_status($timeout = 0): array { + $result = ['connected' => false, 'foundcore' => false]; + try { + $options = []; + if ($timeout) { + $options['connect_timeout'] = $timeout; + $options['read_timeout'] = $timeout; + } + $before = microtime(true); + try { + $response = $this->raw_get_request('admin/cores', $options); + } finally { + $result['time'] = microtime(true) - $before; + } + $status = $response->getStatusCode(); + if ($status !== 200) { + $result['error'] = 'Unsuccessful status code: ' . $status; + return $result; + } + $decoded = json_decode($response->getBody()->getContents()); + if (!$decoded) { + $result['error'] = 'Invalid JSON'; + return $result; + } + // Provided we get some valid JSON then probably Solr exists and is responding. + // Any following errors we don't count as not connected (ERROR display in the check) + // because maybe it happens if Solr changes their JSON format in a future version. + $result['connected'] = true; + if (!property_exists($decoded, 'status')) { + $result['error'] = 'Unexpected JSON: no core status'; + return $result; + } + foreach ($decoded->status as $core) { + $match = false; + if (!property_exists($core, 'name')) { + $result['error'] = 'Unexpected JSON: core has no name'; + return $result; + } + if ($core->name === $this->config->indexname) { + $match = true; + } + if (!$match && property_exists($core, 'cloud')) { + if (!property_exists($core->cloud, 'collection')) { + $result['error'] = 'Unexpected JSON: core cloud has no name'; + return $result; + } + if ($core->cloud->collection === $this->config->indexname) { + $match = true; + } + } + + if ($match) { + $result['foundcore'] = true; + if (!property_exists($core, 'index')) { + $result['error'] = 'Unexpected JSON: core has no index'; + return $result; + } + if (!property_exists($core->index, 'sizeInBytes')) { + $result['error'] = 'Unexpected JSON: core index has no sizeInBytes'; + return $result; + } + $result['indexsize'] = $core->index->sizeInBytes; + return $result; + } + } + $result['error'] = 'Could not find core matching ' . $this->config->indexname;; + return $result; + } catch (\Throwable $t) { + $result['error'] = 'Exception occurred: ' . $t->getMessage(); + $result['exception'] = $t; + return $result; + } + } + /** * Returns the solr client instance. * @@ -1453,23 +1549,128 @@ public function get_curl_object() { } /** - * Return a Moodle url object for the server connection. + * Return a Moodle url object for the raw server URL (containing all indexes). * * @param string $path The solr path to append. * @return \moodle_url */ - public function get_connection_url($path) { + public function get_server_url(string $path): \moodle_url { // Must use the proper protocol, or SSL will fail. $protocol = !empty($this->config->secure) ? 'https' : 'http'; $url = $protocol . '://' . rtrim($this->config->server_hostname, '/'); if (!empty($this->config->server_port)) { $url .= ':' . $this->config->server_port; } - $url .= '/solr/' . $this->config->indexname . '/' . ltrim($path, '/'); - + $url .= '/solr/' . ltrim($path, '/'); return new \moodle_url($url); } + /** + * Return a Moodle url object for the server connection including the search index. + * + * @param string $path The solr path to append. + * @return \moodle_url + */ + public function get_connection_url($path) { + return $this->get_server_url($this->config->indexname . '/' . ltrim($path, '/')); + } + + /** + * Calls the Solr engine with a GET request (for things the Solr extension doesn't support). + * + * This has similar result to get_curl_object but uses the newer (mockable) Guzzle HTTP client. + * + * @param string $path URL path (after /solr/) e.g. 'admin/cores?action=STATUS&core=frog' + * @param array $overrideoptions Optional array of Guzzle options, will override config + * @return \Psr\Http\Message\ResponseInterface Response message from Guzzle + * @throws \GuzzleHttp\Exception\GuzzleException If any problem connecting + * @since Moodle 5.0 + */ + public function raw_get_request( + string $path, + array $overrideoptions = [], + ): \Psr\Http\Message\ResponseInterface { + $client = \core\di::get(\core\http_client::class); + return $client->get( + $this->get_server_url($path)->out(false), + $this->get_http_client_options($overrideoptions), + ); + } + + /** + * Gets the \core\http_client options for a connection. + * + * @param array $overrideoptions Optional array to override some of the options + * @return array Array of http_client options + */ + protected function get_http_client_options(array $overrideoptions = []): array { + $options = [ + 'connect_timeout' => !empty($this->config->server_timeout) ? (int)$this->config->server_timeout : 30, + ]; + $options['read_timeout'] = $options['connect_timeout']; + if (!empty($this->config->server_username)) { + $options['auth'] = [$this->config->server_username, $this->config->server_password]; + } + if (!empty($this->config->ssl_cert)) { + $options['cert'] = $this->config->ssl_cert; + } + if (!empty($this->config->ssl_key)) { + if (!empty($this->config->ssl_keypassword)) { + $options['ssl_key'] = [$this->config->ssl_key, $this->config->ssl_keypassword]; + } else { + $options['ssl_key'] = $this->config->ssl_key; + } + } + if (!empty($this->config->ssl_cainfo)) { + $options['verify'] = $this->config->ssl_cainfo; + } else if (!empty($this->config->ssl_capath)) { + // Guzzle doesn't support a whole path of CA certs, so we have to make a single file + // with all the *.pem files in that directory. It needs to be in filesystem so we can + // use it directly, let's put it in local cache for 10 minutes. + $cachefolder = make_localcache_directory('search_solr'); + $prefix = 'capath.' . sha1($this->config->ssl_capath); + $now = \core\di::get(\core\clock::class)->time(); + $got = false; + foreach (scandir($cachefolder) as $filename) { + // You are not allowed to overwrite files in localcache folders so we use files + // with the time in, and delete old files with a 1 minute delay to avoid race + // conditions. + if (preg_match('~^(.*)\.([0-9]+)$~', $filename, $matches)) { + [1 => $fileprefix, 2 => $time] = $matches; + $pathname = $cachefolder . '/' . $filename; + if ($time > $now - self::CA_PATH_CACHE_TIME && $fileprefix === $prefix) { + $options['verify'] = $pathname; + $got = true; + break; + } else if ($time <= $now - self::CA_PATH_CACHE_TIME - self::CA_PATH_CACHE_DELETE_AFTER) { + unlink($pathname); + } + } + } + + if (!$got) { + // If we don't have it yet, we need to make the cached file. + $allpems = ''; + foreach (scandir($this->config->ssl_capath) as $filename) { + if (preg_match('~\.pem$~', $filename)) { + $pathname = $this->config->ssl_capath . '/' . $filename; + $allpems .= file_get_contents($pathname) . "\n\n"; + } + } + $pathname = $cachefolder . '/' . $prefix . '.' . $now; + file_put_contents($pathname, $allpems); + $options['verify'] = $pathname; + } + } + + // Apply other/overridden options. + foreach ($overrideoptions as $name => $value) { + $options[$name] = $value; + } + + return $options; + } + /** * Solr includes group support in the execute_query function. * diff --git a/search/engine/solr/lang/en/search_solr.php b/search/engine/solr/lang/en/search_solr.php index 4d934dec2a1fb..546cc792b9228 100644 --- a/search/engine/solr/lang/en/search_solr.php +++ b/search/engine/solr/lang/en/search_solr.php @@ -22,6 +22,11 @@ * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ +$string['check_indextoobig'] = 'Index larger than specified size'; +$string['check_nocore'] = 'Cannot find index on Solr server'; +$string['check_nosize'] = 'Unable to determine index size on Solr server'; +$string['check_notconnected'] = 'Cannot connect to Solr server'; +$string['check_time'] = 'Server responded with status in {$a}'; $string['connectionerror'] = 'The specified Solr server is not available or the specified index does not exist'; $string['connectionsettings'] = 'Connection settings'; $string['errorcreatingschema'] = 'Error creating the Solr schema: {$a}'; @@ -32,6 +37,9 @@ $string['fileindexing_help'] = 'If your Solr install supports it, this feature allows Moodle to send files to be indexed.
You will need to reindex all site contents after enabling this option for all files to be added.'; $string['fileindexsettings'] = 'File indexing settings'; +$string['indexsize'] = 'The index is using {$a} on the Solr server.'; +$string['indexsizelimit'] = 'Index size limit'; +$string['indexsizelimit_desc'] = 'Shows an error on the status report page if the search index grows larger than this size (in bytes), and a warning if it exceeds 90%. 0 means no monitoring.'; $string['maxindexfilekb'] = 'Maximum file size to index (kB)'; $string['maxindexfilekb_help'] = 'Files larger than this number of kilobytes will not be included in search indexing. If set to zero, files of any size will be indexed.'; $string['minimumsolr4'] = 'Solr 4.0 is the minimum version required for Moodle'; diff --git a/search/engine/solr/lib.php b/search/engine/solr/lib.php new file mode 100644 index 0000000000000..fa8fcbd82469f --- /dev/null +++ b/search/engine/solr/lib.php @@ -0,0 +1,43 @@ +. + +/** + * Moodle API functions. + * + * @package search_solr + * @copyright 2024 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ + +/** + * Gets status checks contributed by this plugin. + * + * If Solr is enabled and indexing is on, returns a check that the connection works. + * + * @return core\check\check[] Array of status checks + */ +function search_solr_status_checks(): array { + global $CFG; + + // No checks if search engine is not set to Solr, or is disabled. + if (!\core_search\manager::is_indexing_enabled() || $CFG->searchengine !== 'solr') { + return []; + } + + // Since it's turned on and set to Solr, configuration really should be OK and we ought to + // show if it isn't, so turn on the check. + return [new \search_solr\check\connection()]; +} diff --git a/search/engine/solr/settings.php b/search/engine/solr/settings.php index fd566dcd6469a..5ea299f590ae0 100644 --- a/search/engine/solr/settings.php +++ b/search/engine/solr/settings.php @@ -49,6 +49,14 @@ $settings->add(new admin_setting_configtext('search_solr/ssl_cainfo', new lang_string('solrsslcainfo', 'search_solr'), new lang_string('solrsslcainfo_desc', 'search_solr'), '', PARAM_RAW)); $settings->add(new admin_setting_configtext('search_solr/ssl_capath', new lang_string('solrsslcapath', 'search_solr'), new lang_string('solrsslcapath_desc', 'search_solr'), '', PARAM_RAW)); + $settings->add(new admin_setting_configtext( + 'search_solr/indexsizelimit', + new lang_string('indexsizelimit', 'search_solr'), + new lang_string('indexsizelimit_desc', 'search_solr'), + 0, + PARAM_INT, + )); + $settings->add(new admin_setting_heading('search_solr_fileindexing', new lang_string('fileindexsettings', 'search_solr'), '')); $settings->add(new admin_setting_configcheckbox('search_solr/fileindexing', diff --git a/search/engine/solr/tests/engine_test.php b/search/engine/solr/tests/engine_test.php index ace0527f2252e..a88698dc6a7cd 100644 --- a/search/engine/solr/tests/engine_test.php +++ b/search/engine/solr/tests/engine_test.php @@ -1472,6 +1472,19 @@ public function test_add_document_batch_large(): void { $this->assertCount(1, $results); } + /** + * Tests that the get_status function works OK on the real server (there are more detailed + * tests for this function in {@see mock_engine_test}). + * + * @covers \search_solr\check\connection + */ + public function test_get_status(): void { + $status = $this->engine->get_status(5); + $this->assertTrue($status['connected']); + $this->assertTrue($status['foundcore']); + $this->assertGreaterThan(0, $status['indexsize']); + } + /** * Carries out a raw Solr query using the Solr basic query syntax. * diff --git a/search/engine/solr/tests/mock_engine_test.php b/search/engine/solr/tests/mock_engine_test.php new file mode 100644 index 0000000000000..242ad806bd281 --- /dev/null +++ b/search/engine/solr/tests/mock_engine_test.php @@ -0,0 +1,634 @@ +. + +namespace search_solr; + +/** + * Solr search engine unit tests that can operate using a mock http_client and without creating a + * search manager instance. + * + * These tests can run without the solr PHP extension. + * + * All 'realistic' tests of searching (e.g. index something then see if it is found by search) + * require a real Solr instance for testing and should be placed in {@see engine_test}. + * Tests that don't rely heavily on the real search functionality, or where we need to simulate + * multiple different ways of configuring the search infrastructure, or unusual failures in + * communication, may be better suited for this mock test approach. + * + * @package search_solr + * @category test + * @copyright 2024 The Open University + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + * @covers \search_solr\engine + */ +final class mock_engine_test extends \advanced_testcase { + + protected function setUp(): void { + parent::setUp(); + $this->resetAfterTest(); + + // Minimal configuration. + set_config('server_hostname', 'host.invalid', 'search_solr'); + set_config('indexname', 'myindex', 'search_solr'); + + // This is not necessary on my setup but in GitHub Actions, the server_port is set to '' + // instead of default 8983. + set_config('server_port', '8983', 'search_solr'); + } + + /** + * Tests {@see engine::get_server_url}. + */ + public function test_get_server_url(): void { + // Basic URL. + $engine = new engine(); + $this->assertEquals( + 'http://host.invalid:8983/solr/', + $engine->get_server_url('')->out(false), + ); + + // Same but with specified path. + $this->assertEquals( + 'http://host.invalid:8983/solr/twiddle', + $engine->get_server_url('twiddle')->out(false), + ); + // Slash at start of path will be stripped. + $this->assertEquals( + 'http://host.invalid:8983/solr/twiddle', + $engine->get_server_url('/twiddle')->out(false), + ); + + // Turn on https. Due to the way the port setting works, which is bad, this will still have + // the default not-secure port (even though the 'default' on the setting page will now be + // shown as 8443, hmm). User has to change it manually. + set_config('secure', '1', 'search_solr'); + $engine = new engine(); + $this->assertEquals( + 'https://host.invalid:8983/solr/', + $engine->get_server_url('')->out(false), + ); + + // Change port from default. User has to do this manually when enabling secure. + set_config('server_port', '8443', 'search_solr'); + $engine = new engine(); + $this->assertEquals( + 'https://host.invalid:8443/solr/', + $engine->get_server_url('')->out(false), + ); + } + + /** + * Tests {@see engine::get_connection_url}. + */ + public function test_get_connection_url(): void { + // Basic URL. + $engine = new engine(); + $this->assertEquals( + 'http://host.invalid:8983/solr/myindex/', + $engine->get_connection_url('')->out(false), + ); + } + + /** + * Tests {@see engine::raw_get_request()} with no auth settings. + */ + public function test_raw_get_request_no_auth(): void { + $engine = new engine(); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // When there is no auth, there aren't many options, just timeout. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + // Timeout can be changed in config. + set_config('server_timeout', '10', 'search_solr'); + $engine = new engine(); + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 10, + 'read_timeout' => 10, + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + } + + /** + * Tests {@see engine::raw_get_request()} with basic auth settings. + */ + public function test_raw_get_request_basic_auth(): void { + set_config('server_username', 'u', 'search_solr'); + set_config('server_password', 'p', 'search_solr'); + $engine = new engine(); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // Basic auth works with an 'auth' option. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'auth' => ['u', 'p'], + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + } + + /** + * Tests {@see engine::raw_get_request()} with a supplied user certificate. + */ + public function test_raw_get_request_user_cert(): void { + set_config('secure', '1', 'search_solr'); + set_config('ssl_cert', '/tmp/cert.pem', 'search_solr'); + $engine = new engine(); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // User cert auth uses the 'cert' parameter, with or without a key. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'cert' => '/tmp/cert.pem', + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + } + + /** + * Tests {@see engine::raw_get_request()} with a user key (with or without password). + */ + public function test_raw_get_request_user_key(): void { + set_config('secure', '1', 'search_solr'); + set_config('ssl_key', '/tmp/key.pem', 'search_solr'); + $engine = new engine(); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // User cert auth uses the 'cert' parameter, with or without a key. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'ssl_key' => '/tmp/key.pem', + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + set_config('ssl_keypassword', 'frog', 'search_solr'); + $engine = new engine(); + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'ssl_key' => ['/tmp/key.pem', 'frog'], + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + } + + /** + * Tests {@see engine::raw_get_request()} with a certificate bundle for verifying the server. + */ + public function test_raw_get_request_certificate_bundle(): void { + set_config('secure', '1', 'search_solr'); + set_config('ssl_cainfo', '/tmp/allthecerts.pem', 'search_solr'); + $engine = new engine(); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // User cert auth uses the 'cert' parameter, with or without a key. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'verify' => '/tmp/allthecerts.pem', + ], + )->willReturn($response); + $this->assertEquals($response, $engine->raw_get_request('frog')); + } + + /** + * Tests {@see engine::raw_get_request()} with a certificate folder for verifying the server. + * Guzzle doesn't support a certificate folder (curl does) so this code makes a bundle in the + * localcache area. + */ + public function test_raw_get_request_certificate_folder(): void { + global $CFG; + + // Make a directory full of fake .pem files. + $temp = make_request_directory(); + file_put_contents($temp . '/0.pem', "PEM0\n"); + file_put_contents($temp . '/1.pem', "PEM1\n"); + file_put_contents($temp . '/2.txt', "TXT2\n"); + + set_config('secure', '1', 'search_solr'); + set_config('ssl_capath', $temp, 'search_solr'); + + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + + // Party like it's 13 February 2009. + $time = 1234567890; + $this->mock_clock_with_frozen($time); + + // User cert auth uses the 'cert' parameter, with or without a key. + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + + // The filename is the hash of the capath setting plus current time. + $combinedfile = $CFG->dataroot . + '/localcache/search_solr/capath.' . + sha1($temp) . + '.1234567890'; + + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'verify' => $combinedfile, + ], + )->willReturn($response); + $engine = new engine(); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + // Check the file actually is the .pem files concatenated. + $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile)); + + // Let's add another .pem file. + file_put_contents($temp . '/3.pem', "PEM3\n"); + + // 9 minutes 59 seconds later, it will still use the cached version (same file). + $time += 599; + $this->mock_clock_with_frozen($time); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'verify' => $combinedfile, + ], + )->willReturn($response); + $engine = new engine(); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile)); + + // 10 minutes later, it will make a new cached version. + $time += 1; + $this->mock_clock_with_frozen($time); + + $combinedfile2 = $CFG->dataroot . + '/localcache/search_solr/capath.' . + sha1($temp) . + '.1234568490'; + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'verify' => $combinedfile2, + ], + )->willReturn($response); + $engine = new engine(); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + $this->assertEquals("PEM0\n\n\nPEM1\n\n\nPEM3\n\n\n", file_get_contents($combinedfile2)); + + // The old file is still there. + $this->assertEquals("PEM0\n\n\nPEM1\n\n\n", file_get_contents($combinedfile)); + + // Go another minute. We're still using the same combined file... + $time += 60; + $this->mock_clock_with_frozen($time); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'https://host.invalid:8983/solr/frog', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + 'verify' => $combinedfile2, + ], + )->willReturn($response); + $engine = new engine(); + $this->assertEquals($response, $engine->raw_get_request('frog')); + + $this->assertEquals("PEM0\n\n\nPEM1\n\n\nPEM3\n\n\n", file_get_contents($combinedfile2)); + + // But now it will delete the old one. + $this->assertFalse(file_exists($combinedfile)); + } + + /** + * Tests the {@see engine::get_status()} function when there is an exception connecting. + */ + public function test_get_status_exception_connecting(): void { + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willThrowException(new \coding_exception('ex')); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertFalse($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals( + 'Exception occurred: Coding error detected, it must be fixed by a programmer: ex', + $status['error'], + ); + $this->assertInstanceOf(\coding_exception::class, $status['exception']); + } + + /** + * Tests the {@see engine::get_status()} function when the server returns 404. + */ + public function test_get_status_bad_http_status(): void { + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + $response->method('getStatusCode')->willReturn(404); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertFalse($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals('Unsuccessful status code: 404', $status['error']); + } + + /** + * Creates a mock ResponseInterface with a body containing the specified string. + * + * @param string $body Body content + * @return \Psr\Http\Message\ResponseInterface Interface + */ + protected function get_fake_response(string $body): \Psr\Http\Message\ResponseInterface { + $response = $this->createStub(\Psr\Http\Message\ResponseInterface::class); + $response->method('getStatusCode')->willReturn(200); + $stream = $this->createStub(\Psr\Http\Message\StreamInterface::class); + $response->method('getBody')->willReturn($stream); + $stream->method('getContents')->willReturn($body); + return $response; + } + + /** + * Tests the {@see engine::get_status()} function when the server returns invalid JSON. + * In real life this would only be likely to happen if the server is down and a load balancer + * in front of it for some crazy reason interposes a page with status 200. + */ + public function test_get_status_not_json(): void { + $response = $this->get_fake_response('notjson'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertFalse($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals('Invalid JSON', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when the server returns an empty response. + * + * This could maybe happen if the server has been configured, but not fully initialised. + */ + public function test_get_status_no_cores(): void { + $response = $this->get_fake_response('{}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals('Unexpected JSON: no core status', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when the server returns a core without a name + * we can read. + * + * In real usage this should only happen if the Solr REST interface changes unexpectedly. + */ + public function test_get_status_core_no_name(): void { + // A core with no name (in its 'name' field, the 'frog' key is ignored). + $response = $this->get_fake_response('{"status":{"frog":{}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals('Unexpected JSON: core has no name', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when the server doesn't return status for a + * core that matches the index name in Moodle config. + * + * In real usage this could happen if the index got wiped from search or something. + */ + public function test_get_status_no_matching_core(): void { + // Core is not the one we're looking for. + $response = $this->get_fake_response('{"status":{"frog":{"name":"frog"}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertFalse($status['foundcore']); + $this->assertEquals('Could not find core matching myindex', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when the server returns a core without index + * information. + * + * In real usage this should only happen if the Solr REST interface changes unexpectedly. There + * is a parameter to not receive index information, but we don't use it. + */ + public function test_get_status_core_no_index(): void { + // Core exists but has no index object. + $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex"}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertTrue($status['foundcore']); + $this->assertEquals('Unexpected JSON: core has no index', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when the server returns index information + * without size. + * + * In real usage this should only happen if the Solr REST interface changes unexpectedly. + */ + public function test_get_status_core_index_no_size(): void { + // Core index objects doesn't have a size. + $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex","index":{}}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertTrue($status['foundcore']); + $this->assertEquals('Unexpected JSON: core index has no sizeInBytes', $status['error']); + } + + /** + * Tests the {@see engine::get_status()} function when all desired data is present, using a + * single-instance Solr configuration. + */ + public function test_get_status_success_single_server(): void { + // Core index complete with size. + $response = $this->get_fake_response('{"status":{"myindex":{"name":"myindex",' . + '"index":{"sizeInBytes":123}}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertTrue($status['foundcore']); + $this->assertEquals(123, $status['indexsize']); + } + + /** + * Tests the {@see engine::get_status()} function when all desired data is present, using a + * multiple-instance (SolrCloud) configuration. + */ + public function test_get_status_success_solr_cloud(): void { + // Index with size, in cloud replica. These have a different name for each node but a + // 'collection' field with the original index name. + $response = $this->get_fake_response('{"status":{"replica1":{"name":"replica1",' . + '"cloud":{"collection":"myindex"},"index":{"sizeInBytes":123}}}}'); + + $mockedclient = $this->createMock(\core\http_client::class); + \core\di::set(\core\http_client::class, $mockedclient); + $mockedclient->expects($this->once())->method('get')->with( + 'http://host.invalid:8983/solr/admin/cores', + [ + 'connect_timeout' => 30, + 'read_timeout' => 30, + ], + )->willReturn($response); + $engine = new engine(); + $status = $engine->get_status(); + $this->assertTrue($status['connected']); + $this->assertTrue($status['foundcore']); + $this->assertEquals(123, $status['indexsize']); + } +}