[RORDEV-1206] ES 8.14.0 support (#1021)

azure-pipelines.yml

@@ -82,7 +82,7 @@ stages:
               ROR_TASK: core_tests
       - job:
         condition: or(eq(variables.isEpic, true), eq(variables.isDevelop, true), eq(variables.isMaster, true))
-        container: maven:3.8.1-openjdk-17-slim
+        container: openjdk:22-slim
         timeoutInMinutes: 120
         steps:
           - checkout: self
@@ -108,6 +108,8 @@ stages:
         strategy:
           maxParallel: 99
           matrix:
+            IT_es814x:
+              ROR_TASK: integration_es814x
             IT_es813x:
               ROR_TASK: integration_es813x
             IT_es812x:
@@ -136,7 +138,7 @@ stages:
               ROR_TASK: integration_es80x
       - job:
         condition: and(ne(variables.isEpic, true), ne(variables.isDevelop, true), ne(variables.isMaster, true))
-        container: maven:3.8.1-openjdk-17-slim
+        container: openjdk:22-slim
         timeoutInMinutes: 120
         steps:
           - checkout: self
@@ -162,8 +164,8 @@ stages:
         strategy:
           maxParallel: 99
           matrix:
-            IT_es813x:
-              ROR_TASK: integration_es813x
+            IT_es814x:
+              ROR_TASK: integration_es814x
             IT_es80x:
               ROR_TASK: integration_es80x
       - job:

ci/run-pipeline.sh

@@ -50,6 +50,10 @@ run_integration_tests() {
   ./gradlew integration-tests:test "-PesModule=$ES_MODULE" || (find . | grep hs_err | xargs cat && exit 1)
 }
 
+if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "integration_es814x" ]]; then
+  run_integration_tests "es814x"
+fi
+
 if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "integration_es813x" ]]; then
   run_integration_tests "es813x"
 fi

ci/supported-es-versions/es8x.txt

@@ -1,3 +1,4 @@
+8.14.0
 8.13.4
 8.13.3
 8.13.2

development.md

@@ -58,10 +58,9 @@ Currently eshome support debugging only es8x modules.
 * Docker
 
 ## Running tests
-* unit tests: `./gradlew test ror`
+* unit tests: `./gradlew core:test`
 * integration tests for specific module (at the moment we have two modules with integration tests): 
-  * `./gradlew integration-tests:test '-PesModule=es70x'` 
-  * `./gradlew integration-tests-scala:test '-PesModule=es70x'`
+  * `./gradlew integration-tests:test -PesModule=es70x`
 ## Adding license headers to newly created files:
 * `./gradlew licenseFormatMain`
 * `./gradlew licenseFormatTest`

docker-based-builder/build.sh

@@ -10,5 +10,11 @@ fi
 ES_VERSIONS=$*
 
 docker buildx build --no-cache --progress=plain --load -t ror-builder-tmp -f ror-builder/Dockerfile-ror-builder ../
-docker run --rm -v "$(pwd)"/../builds:/ror/builds ror-builder-tmp "$ES_VERSIONS"
-docker rmi -f ror-builder-tmp
+
+cleanup() {
+  docker rmi -f ror-builder-tmp
+}
+
+trap cleanup EXIT
+
+docker run --rm -v "$(pwd)"/../builds:/ror/builds ror-builder-tmp "$ES_VERSIONS"

docker-based-builder/ror-builder/Dockerfile-ror-builder

@@ -1,4 +1,4 @@
-FROM openjdk:21-jdk
+FROM openjdk:22-jdk
 
 COPY ./ /ror
 WORKDIR /ror

es814x/Dockerfile

@@ -0,0 +1,29 @@
+ARG ES_VERSION
+ARG ROR_VERSION
+
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ES_VERSION}
+
+ARG ES_VERSION
+ARG ROR_VERSION
+
+ENV KIBANA_USER_PASS=kibana
+ENV ADMIN_USER_PASS=admin
+
+USER root
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends gosu && \
+    rm -rf /var/lib/apt/lists/* && \
+    gosu nobody true
+
+USER elasticsearch
+
+COPY readonlyrest-${ROR_VERSION}_es${ES_VERSION}.zip /tmp/readonlyrest.zip
+COPY init-readonlyrest.yml /usr/share/elasticsearch/config/readonlyrest.yml
+COPY ror-entrypoint.sh /usr/local/bin/ror-entrypoint.sh
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:///tmp/readonlyrest.zip
+
+USER root
+
+ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/ror-entrypoint.sh"]

es814x/build.gradle

@@ -0,0 +1,165 @@
+/*
+ *    This file is part of ReadonlyREST.
+ *
+ *    ReadonlyREST is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation, either version 3 of the License, or
+ *    (at your option) any later version.
+ *
+ *    ReadonlyREST is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+ */
+import com.bmuschko.gradle.docker.tasks.image.*
+import org.gradle.crypto.checksum.Checksum
+
+plugins {
+    id 'com.bmuschko.docker-remote-api'
+    id 'org.gradle.crypto.checksum'
+    id "readonlyrest.plugin-common-conventions"
+}
+
+def pluginFullName = pluginName + '-' + version
+def projectJavaLanguageVersion = JavaLanguageVersion.of(17)
+def moduleEsVersion = project.property('esVersion') != null ? project.property('esVersion') : project.property('latestSupportedEsVersion')
+def dockerImageLatest = 'beshultd/elasticsearch-readonlyrest:' + moduleEsVersion + '-ror-latest'
+def dockerImageVersion = 'beshultd/elasticsearch-readonlyrest:' + moduleEsVersion + '-ror-' + pluginVersion
+
+java {
+    toolchain {
+        languageVersion = projectJavaLanguageVersion
+    }
+}
+
+dependencies {
+    implementation  project(path: ':core')
+    implementation  project(path: ':ror-tools', configuration: 'shadow')
+    implementation  project(path: ':ror-tools-core')
+    implementation  group: 'org.elasticsearch',                 name: 'elasticsearch' ,                         version: moduleEsVersion
+    implementation  group: 'org.elasticsearch.client',          name: 'elasticsearch-rest-client',              version: moduleEsVersion
+    implementation  group: 'org.apache.httpcomponents',         name: 'httpclient',                             version: '4.5.13'
+    implementation  group: 'org.apache.logging.log4j',          name: 'log4j-core',                             version: '2.11.1'
+    compileOnly     group: 'org.locationtech.spatial4j',        name: 'spatial4j',                              version: '0.7'
+    // if you don't have this dependency in local maven, please run publishToMavenLocal task first
+    compileOnly     group: 'org.elasticsearch.plugin',          name: 'transport-netty4',                       version: moduleEsVersion
+}
+
+configurations {
+    wagon
+    distJars {
+        exclude group: 'org.elasticsearch'
+        exclude group: 'lucene-core'
+        exclude module: 'log4j-api'
+        exclude module: 'log4j-core'
+        exclude group: 'lucene-analyzers-common'
+        exclude group: 'org.apache.commons'
+        exclude group: 'org.yaml'
+        exclude group: 'com.fasterxml.jackson.core', module: 'jackson-core'
+    }
+}
+
+tasks.register('cleanOldData') {
+    doLast {
+        delete 'build/tmp/' + pluginFullName
+    }
+}
+
+tasks.register('jarHellCheck', JavaExec) {
+    outputs.upToDateWhen { false }
+    mainClass.set("org.elasticsearch.jdk.JarHell")
+    classpath = project.sourceSets.main.compileClasspath.filter { it.exists() }
+    javaLauncher = javaToolchains.launcherFor {
+        languageVersion = projectJavaLanguageVersion
+    }
+}
+
+tasks.register('resolvePluginDescriptorTemplate', Copy) {
+    outputs.upToDateWhen { false }
+    from './plugin-metadata'
+    into 'build/tmp/' + pluginFullName
+    expand([
+            'descriptor': [
+                    'name'         : pluginName,
+                    'pluginVersion': pluginVersion,
+                    'esVersion'    : moduleEsVersion
+            ]
+    ])
+}
+
+tasks.register('buildRorPluginZip') {
+    dependsOn(packageRorPlugin, createRorPluginChecksums)
+}
+
+tasks.register('packageRorPlugin', Zip) {
+    dependsOn(cleanOldData, jarHellCheck, toJar, resolvePluginDescriptorTemplate)
+    outputs.upToDateWhen { false }
+    archivesBaseName = pluginName
+    into('.') {
+        from configurations.distJars.filter { x -> !x.name.contains('spatial4j') && !x.name.contains('jts') }
+        from 'build/libs/' + pluginFullName + '.jar'
+        from 'build/tmp/' + pluginFullName
+    }
+}
+
+tasks.register('createRorPluginChecksums', Checksum) {
+    dependsOn(packageRorPlugin)
+    def distributionsDir = layout.buildDirectory.dir("distributions")
+
+    outputs.upToDateWhen { false }
+    inputFiles.setFrom(packageRorPlugin.archiveFile)
+    outputDirectory.set(distributionsDir)
+    checksumAlgorithm.set(Checksum.Algorithm.SHA512)
+}
+
+tasks.register('uploadRorPluginToS3', Exec) {
+    dependsOn(packageRorPlugin, createRorPluginChecksums)
+
+    def distributionsDir = layout.buildDirectory.get().asFile.path + "/distributions"
+    def pluginFileZip = distributionsDir + "/" + pluginFullName + ".zip"
+    def pluginSha1 = distributionsDir + "/" + pluginFullName + ".zip.sha1"
+    def pluginSha512 = distributionsDir + "/" + pluginFullName + ".zip.sha512"
+    def targetS3Dir = "build/" + pluginVersion + "/"
+
+    commandLine '../ci/upload-files-to-s3.sh', pluginFileZip, pluginSha512, pluginSha1, targetS3Dir
+}
+
+tasks.register('prepareDockerImageFiles', Copy) {
+    dependsOn packageRorPlugin
+    outputs.upToDateWhen { false }
+
+    from layout.projectDirectory.file("Dockerfile")
+    from layout.buildDirectory.file("distributions/" + pluginFullName + ".zip")
+    from rootProject.files("docker-image")
+
+    into layout.buildDirectory.dir("docker-image")
+}
+
+tasks.register('buildRorDockerImage', DockerBuildImage) {
+    dependsOn('packageRorPlugin', 'prepareDockerImageFiles')
+
+    inputDir = layout.buildDirectory.dir("docker-image")
+    buildArgs = [
+            ES_VERSION: project.properties['esVersion'],
+            ROR_VERSION: pluginVersion
+    ]
+    images.add(dockerImageLatest)
+    images.add(dockerImageVersion)
+}
+
+tasks.register('removeRorDockerImage', DockerRemoveImage) {
+    dependsOn buildRorDockerImage
+    force = true
+    targetImageId buildRorDockerImage.getImageId()
+}
+
+tasks.register('pushRorDockerImage', DockerPushImage) {
+    dependsOn buildRorDockerImage
+
+    images.add(dockerImageLatest)
+    images.add(dockerImageVersion)
+}
+pushRorDockerImage.configure { finalizedBy removeRorDockerImage }

es814x/gradle.properties

@@ -0,0 +1 @@
+latestSupportedEsVersion=8.14.0

es814x/plugin-metadata/plugin-descriptor.properties

@@ -0,0 +1,27 @@
+#
+# This file is part of ReadonlyREST.
+#
+#     ReadonlyREST is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     ReadonlyREST is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+#
+#
+# Elasticsearch plugin descriptor file
+# This file must exist as 'plugin-descriptor.properties' in a folder named `elasticsearch`
+# inside all plugins.
+#
+name=${descriptor.name}
+version=${descriptor.pluginVersion}
+description=Safely expose Elasticsearch REST API
+classname=tech.beshu.ror.es.ReadonlyRestPlugin
+java.version=1.8
+elasticsearch.version=${descriptor.esVersion}

es814x/plugin-metadata/plugin-security.policy

@@ -0,0 +1,14 @@
+grant {
+  permission java.io.FilePermission "/usr/share/elasticsearch", "read";
+  permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";
+  permission java.lang.RuntimePermission "accessClassInPackage.sun.misc.*";
+  permission java.lang.RuntimePermission "accessDeclaredMembers";
+  permission java.lang.RuntimePermission "getClassLoader";
+  permission java.lang.RuntimePermission "setContextClassLoader";
+  permission java.net.SocketPermission "*", "accept, resolve, connect";
+  permission java.security.SecurityPermission "insertProvider";
+  permission java.security.SecurityPermission "putProviderProperty.BCFIPS";
+  permission java.security.SecurityPermission "putProviderProperty.BCJSSE";
+  permission java.util.PropertyPermission "*", "read,write";
+};