diff --git a/pom.xml b/pom.xml index c8c12067f7..c3d5733c8c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,12 +6,12 @@ org.gbif motherpom - 56 + 59 org.gbif.registry registry-parent - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT GBIF Registry Parent GBIF Registry project @@ -46,7 +46,7 @@ https://repository.gbif.org/content/repositories/releases - gbif-snapshot + gbif-deploy https://repository.gbif.org/content/repositories/snapshots @@ -74,31 +74,35 @@ + 11 + 11 + 11 + 2.3.12.RELEASE 2.2.4 - 2.2.6.RELEASE + 2.2.10.RELEASE 2.2.5.RELEASE - 2.1.0 + 2.3.2 1.5.8 2.2.8.RELEASE - 1.16.18 + 2.0.0 0.59 - 1.4 - 1.26 + 2.0.0 + 2.0.0 1.13 - 2.17 - 1.53.3 + 3.0.0 + 2.0.0 0.10 2.6 0.67 1.8.1 4.1.0-gbif-4 - 0.195.1 - 1.0.12 - 2.128 + 0.195.8 + 2.0.2 + 3.0.0 3.2 @@ -109,7 +113,7 @@ 7.9.1 10.11 3.0.1 - 18.0 + 27.0-jre 2.6.0-cdh5.16.2 3.2.0 2.5.10 @@ -118,19 +122,19 @@ 1.1.1 1.2.3 4.11 - 1.18.20 + 1.18.28 3.5.0 - 4.9.1 - 2.10.0 - 2.8.0 1.7.30 4.4.1 - 3.4.14 + 3.8.1 + 5.5.0 20200713.1 2.4.0 5.7.1 1.14 1.8.2 + 2.3.1 + 2.2.3 1.7.0 @@ -141,8 +145,8 @@ 5.9.1 3.3.3 2.6.3 - 1.17.6 - 1.17.6 + 1.19.8 + 1.19.8 1.9 2.0.0 0.9.12 @@ -168,6 +172,7 @@ ${spring-boot.version} + 3.5.3 @@ -451,6 +456,21 @@ + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.curator + curator-x-discovery + ${curator.version} + org.mybatis mybatis @@ -529,42 +549,6 @@ ${geojson.version} - - com.squareup.retrofit2 - retrofit - ${retrofit.version} - - - com.squareup.retrofit2 - converter-jackson - ${retrofit.version} - - - com.squareup.retrofit2 - converter-simplexml - ${retrofit.version} - - - com.squareup.retrofit2 - converter-scalars - ${retrofit.version} - - - com.squareup.okhttp3 - okhttp - ${okhttp.version} - - - com.squareup.okhttp3 - logging-interceptor - ${okhttp.version} - - - com.squareup.okio - okio - ${okio.version} - runtime - com.googlecode.owasp-java-html-sanitizer owasp-java-html-sanitizer @@ -577,16 +561,6 @@ ${janino-logback.version} runtime - - - - - - - - - - net.logstash.logback logstash-logback-encoder @@ -736,6 +710,18 @@ + + javax.xml.bind + jaxb-api + ${jaxb-api.version} + + + + io.swagger.core.v3 + swagger-annotations + ${swagger-annotations.version} + + org.mybatis.spring.boot @@ -826,6 +812,11 @@ spring-boot-maven-plugin ${spring-boot-maven-plugin.version} + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + diff --git a/registry-cli/pom.xml b/registry-cli/pom.xml index bac3f07176..5bb6deac9c 100644 --- a/registry-cli/pom.xml +++ b/registry-cli/pom.xml @@ -4,7 +4,7 @@ org.gbif.registry registry-parent - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT registry-cli @@ -19,14 +19,17 @@ + 11 + 11 + 11 ${project.parent.basedir} 1.48 2.2.0 - 1.0 - 1.5 + 1.2-SNAPSHOT + 1.11 1.19.3 1.4.7 - 18.0 + 1.4.20 ${project.basedir}/../target/site/jacoco-it/jacoco.xml,${project.basedir}/../target/site/jacoco-ut/jacoco.xml @@ -53,8 +56,8 @@ maven-compiler-plugin - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} true true @@ -113,18 +116,6 @@ - @@ -139,11 +130,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - - org.jacoco - jacoco-maven-plugin - @@ -185,11 +171,6 @@ spring-jdbc - - com.squareup.retrofit2 - converter-jackson - - org.gbif @@ -339,11 +320,6 @@ com.google.code.findbugs jsr305 - - com.squareup.okhttp3 - okhttp - runtime - @@ -375,6 +351,23 @@ mail ${javax.mail.version} + + org.apache.ant + ant + 1.10.14 + + + + + org.jetbrains.kotlin + kotlin-stdlib-common + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-stdlib + ${kotlin.version} + diff --git a/registry-cli/src/main/java/org/gbif/registry/cli/common/spring/SpringContextBuilder.java b/registry-cli/src/main/java/org/gbif/registry/cli/common/spring/SpringContextBuilder.java index 4393a8c11f..f5ef5d77e5 100644 --- a/registry-cli/src/main/java/org/gbif/registry/cli/common/spring/SpringContextBuilder.java +++ b/registry-cli/src/main/java/org/gbif/registry/cli/common/spring/SpringContextBuilder.java @@ -13,6 +13,11 @@ */ package org.gbif.registry.cli.common.spring; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.collect.ImmutableMap; import org.gbif.api.ws.mixin.Mixins; import org.gbif.common.messaging.ConnectionParameters; import org.gbif.common.messaging.DefaultMessagePublisher; @@ -36,12 +41,6 @@ import org.gbif.ws.security.Md5EncodeServiceImpl; import org.gbif.ws.security.SecretKeySigningService; import org.gbif.ws.security.SigningService; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - import org.mybatis.spring.annotation.MapperScan; import org.mybatis.spring.boot.autoconfigure.MybatisAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration; @@ -56,11 +55,10 @@ import org.springframework.context.annotation.FilterType; import org.springframework.core.env.MapPropertySource; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; /** Utility class to create Spring contexts to be used later in CLI applications. */ public class SpringContextBuilder { @@ -214,6 +212,15 @@ public AnnotationConfigApplicationContext build() { ctx.register(UpdateDownloadStatsService.class); } + ctx.getEnvironment() + .getPropertySources() + .addLast( + new MapPropertySource( + "ManagedProperties", + new ImmutableMap.Builder() + .put("spring.cloud.compatibility-verifier.enabled", "false") + .build())); + if (componentClasses != null) { for (Class c : componentClasses) { ctx.register(c); diff --git a/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java b/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java index d712426589..44e0875072 100644 --- a/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java +++ b/registry-cli/src/main/java/org/gbif/registry/cli/datasetindex/SpringContextBuilder.java @@ -13,6 +13,13 @@ */ package org.gbif.registry.cli.datasetindex; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.DateDeserializers; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableMap; +import com.zaxxer.hikari.HikariDataSource; import org.gbif.api.service.registry.DatasetService; import org.gbif.api.service.registry.InstallationService; import org.gbif.api.service.registry.NetworkService; @@ -35,10 +42,7 @@ import org.gbif.registry.ws.client.OrganizationClient; import org.gbif.ws.client.ClientBuilder; import org.gbif.ws.json.JacksonJsonObjectMapperProvider; - -import java.io.IOException; -import java.util.Date; - +import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.boot.actuate.autoconfigure.elasticsearch.ElasticSearchRestHealthContributorAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration; @@ -54,13 +58,8 @@ import org.springframework.context.annotation.Primary; import org.springframework.core.env.MapPropertySource; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.deser.std.DateDeserializers; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.common.collect.ImmutableMap; -import com.zaxxer.hikari.HikariDataSource; +import java.io.IOException; +import java.util.Date; public class SpringContextBuilder { @@ -148,6 +147,7 @@ private static AnnotationConfigApplicationContext commonContext( configuration.getClbDb().getConnectionTimeout()) .put("indexing.stopAfter", configuration.getStopAfter()) .put("indexing.pageSize", configuration.getPageSize()) + .put("spring.cloud.compatibility-verifier.enabled", "false") .build())); ctx.refresh(); ctx.start(); @@ -199,13 +199,13 @@ public HikariDataSource clbDataSource(DatasetIndexConfiguration configuration) { public ClientBuilder clientBuilder(DatasetIndexConfiguration configuration) { ClientBuilder clientBuilder = new ClientBuilder(); - ObjectMapper objectMapper = JacksonJsonObjectMapperProvider.getObjectMapperWithBuilderSupport(); + ObjectMapper objectMapper = + JacksonJsonObjectMapperProvider.getObjectMapperWithBuilderSupport(); SimpleModule simpleModule = new SimpleModule(); simpleModule.addDeserializer(Date.class, new CustomDateDeserializer()); objectMapper.registerModule(simpleModule); - clientBuilder.withObjectMapper( - objectMapper); + clientBuilder.withObjectMapper(objectMapper); clientBuilder.withUrl( configuration.getRegistryWsUrl() != null ? configuration.getRegistryWsUrl() @@ -234,7 +234,7 @@ public NetworkService networkService(ClientBuilder clientBuilder) { } @Bean - public GithubApiClient githubApiClient(ClientBuilder clientBuilder){ + public GithubApiClient githubApiClient(ClientBuilder clientBuilder) { return clientBuilder.build(GithubApiClient.class); } } diff --git a/registry-directory/pom.xml b/registry-directory/pom.xml index a9dd281509..21668f8a62 100644 --- a/registry-directory/pom.xml +++ b/registry-directory/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-doi/generate-doi-for-old-download b/registry-doi/generate-doi-for-old-download index ea360c2a56..85ae213ed6 100755 --- a/registry-doi/generate-doi-for-old-download +++ b/registry-doi/generate-doi-for-old-download @@ -59,6 +59,9 @@ date=$(curl -Ss https://api.gbif.org/v1/occurrence/download/$downloadKey | jq -r echo "COMMIT;" ) | tee | toDb +curl -X BAN --fail -i https://api.gbif.org -H 'X-Ban-URL: '$downloadKey +sleep 1 + newDoi=$(curl -Ss https://api.gbif.org/v1/occurrence/download/$downloadKey | jq -r .doi) ssh crap@prodcrawler1-vh.gbif.org ./util/doi-synchronizer --fix-doi --doi $newDoi --skip-dia diff --git a/registry-doi/pom.xml b/registry-doi/pom.xml index 0f0a2d11be..51e735caea 100644 --- a/registry-doi/pom.xml +++ b/registry-doi/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-domain/pom.xml b/registry-domain/pom.xml index 5c37b4cabe..72e8ee3761 100644 --- a/registry-domain/pom.xml +++ b/registry-domain/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyDataset.java b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyDataset.java index 0181739c9c..0b7f3b0d17 100644 --- a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyDataset.java +++ b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyDataset.java @@ -13,6 +13,12 @@ */ package org.gbif.registry.domain.ws; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import org.apache.commons.validator.routines.EmailValidator; import org.gbif.api.annotation.Generated; import org.gbif.api.annotation.ParamName; import org.gbif.api.model.common.DOI; @@ -21,33 +27,21 @@ import org.gbif.api.model.registry.Dataset; import org.gbif.api.model.registry.Endpoint; import org.gbif.api.util.VocabularyUtils; -import org.gbif.api.vocabulary.ContactType; -import org.gbif.api.vocabulary.DatasetSubtype; -import org.gbif.api.vocabulary.DatasetType; -import org.gbif.api.vocabulary.EndpointType; -import org.gbif.api.vocabulary.Language; +import org.gbif.api.vocabulary.*; import org.gbif.registry.domain.ws.util.LegacyResourceConstants; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Set; -import java.util.StringTokenizer; -import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlTransient; - -import org.apache.commons.validator.routines.EmailValidator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Objects; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Set; +import java.util.StringTokenizer; +import java.util.UUID; /** * Class used to create or update a Dataset for legacy (GBRDS/IPT) API. Previously known as a @@ -94,9 +88,10 @@ public class LegacyDataset extends Dataset implements LegacyEntity { LegacyResourceConstants.MATERIAL_ENTITY_SERVICE_TYPE_1, LegacyResourceConstants.MATERIAL_ENTITY_SERVICE_TYPE_2, LegacyResourceConstants.SAMPLING_EVENT_SERVICE_TYPE); - private static final Set DATA_PACKAGE_ENDPOINT_TYPE_ALTERNATIVES = + private static final Set CAMTRAP_DP_ENDPOINT_TYPE_ALTERNATIVES = ImmutableSet.of( - EndpointType.CAMTRAP_DP.name(), LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE); + LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE_1, + LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE_2); /** Default constructor. */ public LegacyDataset() { @@ -756,15 +751,26 @@ private void addDataPackageEndpoint() { while (serviceTypesTokenizer.hasMoreTokens() && serviceUrlsTokenizer.hasMoreTokens()) { String type = serviceTypesTokenizer.nextToken(); String url = serviceUrlsTokenizer.nextToken(); - if (type != null && url != null && DATA_PACKAGE_ENDPOINT_TYPE_ALTERNATIVES.contains(type)) { - // create endpoint - Endpoint endpoint = createEndpoint(url, EndpointType.CAMTRAP_DP); - if (endpoint != null) { - // set it - dataPackageEndpoint = endpoint; + if (type != null && url != null) { + if (CAMTRAP_DP_ENDPOINT_TYPE_ALTERNATIVES.contains(type)) { + // create Camtrap DP endpoint + Endpoint endpoint = createEndpoint(url, EndpointType.CAMTRAP_DP); + if (endpoint != null) { + // set it + dataPackageEndpoint = endpoint; + } + // only 1 is expected + break; + } else if (LegacyResourceConstants.COLDP_SERVICE_TYPE.equals(type)) { + // create ColDP enpoint + Endpoint endpoint = createEndpoint(url, EndpointType.COLDP); + if (endpoint != null) { + // set it + dataPackageEndpoint = endpoint; + } + // only 1 is expected + break; } - // only 1 is expected - break; } } } @@ -772,7 +778,7 @@ private void addDataPackageEndpoint() { /** * Creates a new Endpoint, or retrieves an existing Endpoint from the dataset. This method assumes - * that the dataset only has 1 endpoint for each type: EML and DWC_ARCHIVE. + * that the dataset only has 1 endpoint for each type: EML, DWC_ARCHIVE, CAMTRAP_DP or COLDP. * * @param url Endpoint URL * @param type Endpoint type @@ -810,11 +816,13 @@ private Endpoint createEndpoint(String url, EndpointType type) { public DatasetType resolveType() { if (serviceTypes != null) { if (serviceTypes.contains(LegacyResourceConstants.CHECKLIST_SERVICE_TYPE_1) - || serviceTypes.contains(LegacyResourceConstants.CHECKLIST_SERVICE_TYPE_2)) { + || serviceTypes.contains(LegacyResourceConstants.CHECKLIST_SERVICE_TYPE_2) + || serviceTypes.contains(LegacyResourceConstants.COLDP_SERVICE_TYPE)) { return DatasetType.CHECKLIST; } else if (serviceTypes.contains(LegacyResourceConstants.OCCURRENCE_SERVICE_TYPE_1) || serviceTypes.contains(LegacyResourceConstants.OCCURRENCE_SERVICE_TYPE_2) - || serviceTypes.contains(LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE)) { + || serviceTypes.contains(LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE_1) + || serviceTypes.contains(LegacyResourceConstants.CAMTRAP_DP_SERVICE_TYPE_2)) { return DatasetType.OCCURRENCE; } else if (serviceTypes.contains(LegacyResourceConstants.MATERIAL_ENTITY_SERVICE_TYPE_1) || serviceTypes.contains(LegacyResourceConstants.MATERIAL_ENTITY_SERVICE_TYPE_2)) { @@ -909,7 +917,7 @@ public int hashCode() { @Generated @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("primaryContactType", primaryContactType) .add("primaryContactEmail", primaryContactEmail) .add("primaryContactName", primaryContactName) diff --git a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyEndpoint.java b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyEndpoint.java index fb0901a1b4..03de2a5d8f 100644 --- a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyEndpoint.java +++ b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyEndpoint.java @@ -13,26 +13,24 @@ */ package org.gbif.registry.domain.ws; +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.base.Strings; import org.gbif.api.annotation.Generated; import org.gbif.api.annotation.ParamName; import org.gbif.api.model.registry.Endpoint; import org.gbif.api.vocabulary.EndpointType; import org.gbif.registry.domain.ws.util.LegacyResourceConstants; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.UUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.Objects; -import com.google.common.base.Strings; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.UUID; /** * Class used to create or update an Endpoint for legacy (GBRDS/IPT) API. Previously known as a @@ -195,6 +193,6 @@ public int hashCode() { @Generated @Override public String toString() { - return Objects.toStringHelper(this).add("datasetKey", datasetKey).toString(); + return MoreObjects.toStringHelper(this).add("datasetKey", datasetKey).toString(); } } diff --git a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyOrganizationBriefResponse.java b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyOrganizationBriefResponse.java index a8e1926300..552547babb 100644 --- a/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyOrganizationBriefResponse.java +++ b/registry-domain/src/main/java/org/gbif/registry/domain/ws/LegacyOrganizationBriefResponse.java @@ -13,6 +13,7 @@ */ package org.gbif.registry.domain.ws; +import com.google.common.base.MoreObjects; import org.gbif.api.annotation.Generated; import org.gbif.registry.domain.ws.util.LegacyResourceConstants; @@ -81,7 +82,7 @@ public int hashCode() { @Generated @Override public String toString() { - return Objects.toStringHelper(this).add("key", key).add("name", name).toString(); + return MoreObjects.toStringHelper(this).add("key", key).add("name", name).toString(); } public static class LegacyOrganizationArraySerializer diff --git a/registry-domain/src/main/java/org/gbif/registry/domain/ws/util/LegacyResourceConstants.java b/registry-domain/src/main/java/org/gbif/registry/domain/ws/util/LegacyResourceConstants.java index 9408756080..502448fe80 100644 --- a/registry-domain/src/main/java/org/gbif/registry/domain/ws/util/LegacyResourceConstants.java +++ b/registry-domain/src/main/java/org/gbif/registry/domain/ws/util/LegacyResourceConstants.java @@ -61,7 +61,9 @@ public class LegacyResourceConstants { public static final String MATERIAL_ENTITY_SERVICE_TYPE_1 = "DWC-ARCHIVE-MATERIAL-ENTITY"; public static final String MATERIAL_ENTITY_SERVICE_TYPE_2 = "DWC_ARCHIVE_MATERIAL_ENTITY"; public static final String SAMPLING_EVENT_SERVICE_TYPE = "DWC-ARCHIVE-SAMPLING-EVENT"; - public static final String CAMTRAP_DP_SERVICE_TYPE = "CAMTRAP_DP"; + public static final String CAMTRAP_DP_SERVICE_TYPE_1 = "CAMTRAP_DP"; + public static final String CAMTRAP_DP_SERVICE_TYPE_2 = "CAMTRAP-DP"; + public static final String COLDP_SERVICE_TYPE = "COLDP"; // web service paging request size public static final int WS_PAGE_SIZE = 100; diff --git a/registry-events/pom.xml b/registry-events/pom.xml index c7bba82846..f5d8260c06 100644 --- a/registry-events/pom.xml +++ b/registry-events/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-examples/pom.xml b/registry-examples/pom.xml index 18ba2b7a5a..582b708799 100644 --- a/registry-examples/pom.xml +++ b/registry-examples/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-identity/pom.xml b/registry-identity/pom.xml index 5d76bf77d7..54a31c14f8 100644 --- a/registry-identity/pom.xml +++ b/registry-identity/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-identity/src/main/java/org/gbif/registry/identity/model/UserModelMutationResult.java b/registry-identity/src/main/java/org/gbif/registry/identity/model/UserModelMutationResult.java index 09edfb4153..f557f89518 100644 --- a/registry-identity/src/main/java/org/gbif/registry/identity/model/UserModelMutationResult.java +++ b/registry-identity/src/main/java/org/gbif/registry/identity/model/UserModelMutationResult.java @@ -13,6 +13,7 @@ */ package org.gbif.registry.identity.model; +import com.google.common.base.MoreObjects; import org.gbif.api.model.common.AbstractGbifUser; import java.util.HashMap; @@ -127,7 +128,7 @@ public boolean containsError() { @Override public String toString() { - return Objects.toStringHelper(this) + return MoreObjects.toStringHelper(this) .add("username", username) .add("email", email) .add("modelError", modelError) diff --git a/registry-integration-tests/pom.xml b/registry-integration-tests/pom.xml index f6b4021035..dfaebeed26 100644 --- a/registry-integration-tests/pom.xml +++ b/registry-integration-tests/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 @@ -12,6 +12,10 @@ ${project.parent.basedir} + + 11 + 11 + 11 @@ -76,11 +80,6 @@ lombok provided - - com.squareup.retrofit2 - retrofit-mock - test - org.testcontainers elasticsearch diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/resource/CollectionsSearchResourceTest.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/resource/CollectionsSearchResourceTest.java index aae489dca7..ccd6572b0f 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/resource/CollectionsSearchResourceTest.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/resource/CollectionsSearchResourceTest.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.UUID; import org.gbif.api.model.collections.request.CollectionDescriptorsSearchRequest; import org.gbif.api.model.collections.request.InstitutionFacetedSearchRequest; @@ -128,7 +129,7 @@ public void searchCollectionsTest() { public void facetSearchInstitutionsTest() { InstitutionFacetedSearchRequest searchRequest = InstitutionFacetedSearchRequest.builder() - .facets(Collections.singleton(InstitutionFacetParameter.COUNTRY)) + .facets(Set.of(InstitutionFacetParameter.COUNTRY)) .offset(0L) .limit(20) .build(); @@ -140,7 +141,7 @@ public void facetSearchInstitutionsTest() { CollectionFacet facet = new CollectionFacet<>(); facet.setCardinality(1); facet.setField(InstitutionFacetParameter.COUNTRY); - facet.setCounts(Collections.singletonList(new CollectionFacet.Count("ES", 2L))); + facet.setCounts(List.of(new CollectionFacet.Count("ES", 2L))); FacetedSearchResponse searchResponse = new FacetedSearchResponse<>( @@ -163,7 +164,7 @@ public void facetSearchInstitutionsTest() { public void facetSearchCollectionsTest() { CollectionDescriptorsSearchRequest searchRequest = CollectionDescriptorsSearchRequest.builder() - .facets(Collections.singleton(CollectionFacetParameter.COUNTRY)) + .facets(Set.of(CollectionFacetParameter.COUNTRY)) .offset(0L) .limit(20) .build(); @@ -175,7 +176,7 @@ public void facetSearchCollectionsTest() { CollectionFacet facet = new CollectionFacet<>(); facet.setCardinality(1); facet.setField(CollectionFacetParameter.COUNTRY); - facet.setCounts(Collections.singletonList(new CollectionFacet.Count("ES", 2L))); + facet.setCounts(List.of(new CollectionFacet.Count("ES", 2L))); FacetedSearchResponse searchResponse = new FacetedSearchResponse<>( diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/CollectionsSearchIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/CollectionsSearchIT.java index 8ffe57fc5a..f8d72522d7 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/CollectionsSearchIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/CollectionsSearchIT.java @@ -21,10 +21,9 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import lombok.SneakyThrows; import org.gbif.api.model.collections.Address; import org.gbif.api.model.collections.AlternativeCode; @@ -38,7 +37,6 @@ import org.gbif.api.model.collections.search.FacetedSearchResponse; import org.gbif.api.model.collections.search.InstitutionSearchResponse; import org.gbif.api.model.common.export.ExportFormat; -import org.gbif.api.model.common.paging.Pageable; import org.gbif.api.model.common.paging.PagingRequest; import org.gbif.api.model.common.paging.PagingResponse; import org.gbif.api.model.registry.Identifier; @@ -637,10 +635,9 @@ public void collectionsFacetTest() { searchService.searchCollections( CollectionDescriptorsSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - CollectionFacetParameter.COUNTRY, - CollectionFacetParameter.PRESERVATION_TYPE))) + Set.of( + CollectionFacetParameter.COUNTRY, + CollectionFacetParameter.PRESERVATION_TYPE)) .build()); assertEquals(2, searchResponse.getFacets().size()); assertEquals( @@ -658,27 +655,24 @@ public void collectionsFacetTest() { searchService.searchCollections( CollectionDescriptorsSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - CollectionFacetParameter.COUNTRY, - CollectionFacetParameter.PRESERVATION_TYPE))) + Set.of( + CollectionFacetParameter.COUNTRY, + CollectionFacetParameter.PRESERVATION_TYPE)) .facetLimit(1) .build()); assertEquals(2, searchResponse.getFacets().size()); assertTrue(searchResponse.getFacets().stream().allMatch(f -> f.getCounts().size() == 1)); - Map facetPages = new HashMap<>(); - facetPages.put(CollectionFacetParameter.PRESERVATION_TYPE, new PagingRequest(0, 2)); searchResponse = searchService.searchCollections( CollectionDescriptorsSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - CollectionFacetParameter.COUNTRY, - CollectionFacetParameter.PRESERVATION_TYPE))) + Set.of( + CollectionFacetParameter.COUNTRY, + CollectionFacetParameter.PRESERVATION_TYPE)) .facetLimit(1) - .facetPages(facetPages) + .facetPages( + Map.of(CollectionFacetParameter.PRESERVATION_TYPE, new PagingRequest(0, 2))) .build()); assertEquals(2, searchResponse.getFacets().size()); assertEquals( @@ -686,18 +680,16 @@ public void collectionsFacetTest() { assertEquals( 1, searchResponse.getFacets().stream().filter(f -> f.getCounts().size() == 2).count()); - facetPages = new HashMap<>(); - facetPages.put(CollectionFacetParameter.PRESERVATION_TYPE, new PagingRequest(1, 2)); searchResponse = searchService.searchCollections( CollectionDescriptorsSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - CollectionFacetParameter.COUNTRY, - CollectionFacetParameter.PRESERVATION_TYPE))) + Set.of( + CollectionFacetParameter.COUNTRY, + CollectionFacetParameter.PRESERVATION_TYPE)) .facetLimit(1) - .facetPages(facetPages) + .facetPages( + Map.of(CollectionFacetParameter.PRESERVATION_TYPE, new PagingRequest(1, 2))) .build()); assertEquals(2, searchResponse.getFacets().size()); assertTrue(searchResponse.getFacets().stream().allMatch(f -> f.getCounts().size() == 1)); @@ -778,10 +770,7 @@ public void institutionFacetsTest() { searchService.searchInstitutions( InstitutionFacetedSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - InstitutionFacetParameter.DISCIPLINE, - InstitutionFacetParameter.COUNTRY))) + Set.of(InstitutionFacetParameter.DISCIPLINE, InstitutionFacetParameter.COUNTRY)) .multiSelectFacets(true) .build()); assertEquals(2, searchResponse.getFacets().size()); @@ -800,27 +789,19 @@ public void institutionFacetsTest() { searchService.searchInstitutions( InstitutionFacetedSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - InstitutionFacetParameter.COUNTRY, - InstitutionFacetParameter.DISCIPLINE))) + Set.of(InstitutionFacetParameter.COUNTRY, InstitutionFacetParameter.DISCIPLINE)) .facetLimit(1) .build()); assertEquals(2, searchResponse.getFacets().size()); assertTrue(searchResponse.getFacets().stream().allMatch(f -> f.getCounts().size() == 1)); - Map facetPages = new HashMap<>(); - facetPages.put(InstitutionFacetParameter.DISCIPLINE, new PagingRequest(0, 2)); searchResponse = searchService.searchInstitutions( InstitutionFacetedSearchRequest.builder() .facets( - new HashSet<>( - Arrays.asList( - InstitutionFacetParameter.COUNTRY, - InstitutionFacetParameter.DISCIPLINE))) + Set.of(InstitutionFacetParameter.COUNTRY, InstitutionFacetParameter.DISCIPLINE)) .facetLimit(1) - .facetPages(facetPages) + .facetPages(Map.of(InstitutionFacetParameter.DISCIPLINE, new PagingRequest(0, 2))) .build()); assertEquals(2, searchResponse.getFacets().size()); assertEquals( diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/suggestions/CollectionChangeSuggestionServiceIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/suggestions/CollectionChangeSuggestionServiceIT.java index 8bd83475b1..48c34ea6d0 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/suggestions/CollectionChangeSuggestionServiceIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/collections/service/suggestions/CollectionChangeSuggestionServiceIT.java @@ -18,6 +18,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import org.gbif.api.model.collections.Address; +import static org.gbif.registry.service.collections.utils.MasterSourceUtils.IH_SYNC_USER; + import org.gbif.api.model.collections.AlternativeCode; import org.gbif.api.model.collections.Collection; import org.gbif.api.model.collections.Contact; @@ -35,7 +37,6 @@ import org.gbif.api.vocabulary.collections.IdType; import org.gbif.api.vocabulary.collections.MasterSourceType; import org.gbif.registry.service.collections.suggestions.CollectionChangeSuggestionService; -import static org.gbif.registry.service.collections.utils.MasterSourceUtils.IH_SYNC_USER; import org.gbif.ws.client.filter.SimplePrincipalProvider; import java.net.URI; diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java index 0d97b5d951..dad6e85f23 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/CollectionMapperIT.java @@ -306,7 +306,7 @@ public void listTest() { .page(page) .build(), 0); - assertSearch(CollectionListParams.builder().city(asList("Odense")).page(page).build(), 1); + assertSearch(CollectionListParams.builder().city(asList("Odense")).page(page).build(), 0); assertSearch( CollectionListParams.builder() .city(asList("Copenhagen")) diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java index 5d6feaf97f..92634512c3 100644 --- a/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/it/persistence/mapper/InstitutionMapperIT.java @@ -263,7 +263,7 @@ public void listTest() { .page(page) .build(), 0); - assertSearch(InstitutionListParams.builder().city(asList("Odense")).page(page).build(), 1); + assertSearch(InstitutionListParams.builder().city(asList("Odense")).page(page).build(), 0); assertSearch( InstitutionListParams.builder() .city(asList("Copenhagen")) diff --git a/registry-integration-tests/src/test/java/org/gbif/registry/ws/provider/CollectionsSearchRequestHandlerMethodArgumentResolverTest.java b/registry-integration-tests/src/test/java/org/gbif/registry/ws/provider/CollectionsSearchRequestHandlerMethodArgumentResolverTest.java new file mode 100644 index 0000000000..440395805a --- /dev/null +++ b/registry-integration-tests/src/test/java/org/gbif/registry/ws/provider/CollectionsSearchRequestHandlerMethodArgumentResolverTest.java @@ -0,0 +1,274 @@ +package org.gbif.registry.ws.provider; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.gbif.api.model.collections.request.CollectionDescriptorsSearchRequest; +import org.gbif.api.model.collections.request.CollectionSearchRequest; +import org.gbif.api.model.collections.request.FacetedSearchRequest; +import org.gbif.api.model.collections.request.InstitutionFacetedSearchRequest; +import org.gbif.api.model.collections.request.InstitutionSearchRequest; +import org.gbif.api.model.collections.request.SearchRequest; +import org.gbif.api.vocabulary.Country; +import org.gbif.api.vocabulary.GbifRegion; +import org.gbif.api.vocabulary.IdentifierType; +import org.gbif.api.vocabulary.Rank; +import org.gbif.api.vocabulary.SortOrder; +import org.gbif.api.vocabulary.collections.CollectionFacetParameter; +import org.gbif.api.vocabulary.collections.CollectionsFacetParameter; +import org.gbif.api.vocabulary.collections.CollectionsSortField; +import org.gbif.api.vocabulary.collections.InstitutionFacetParameter; +import org.gbif.api.vocabulary.collections.MasterSourceType; +import org.gbif.api.vocabulary.collections.Source; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.core.MethodParameter; +import org.springframework.web.bind.support.WebDataBinderFactory; +import org.springframework.web.context.request.NativeWebRequest; +import org.springframework.web.method.support.ModelAndViewContainer; + +@ExtendWith(MockitoExtension.class) +public class CollectionsSearchRequestHandlerMethodArgumentResolverTest { + + @Mock private NativeWebRequest webRequest; + @Mock private WebDataBinderFactory binderFactory; + @Mock private ModelAndViewContainer mavContainer; + @Mock private MethodParameter methodParameter; + + private final InstitutionSearchRequestHandlerMethodArgumentResolver instResolver = + new InstitutionSearchRequestHandlerMethodArgumentResolver(); + private final CollectionSearchRequestHandlerMethodArgumentResolver collResolver = + new CollectionSearchRequestHandlerMethodArgumentResolver(); + + @Test + public void commonParamsTest() { + mockCommonParams(); + InstitutionSearchRequest institutionSearchRequest = + (InstitutionSearchRequest) + instResolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertCommonParams(institutionSearchRequest); + + mockCommonParams(); + CollectionSearchRequest collectionSearchRequest = + (CollectionSearchRequest) + collResolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertCommonParams(collectionSearchRequest); + } + + @Test + public void institutionParamsTest() { + Map parameterMap = new HashMap<>(); + parameterMap.put("type", new String[] {"t1", "t2"}); + parameterMap.put("institutionalGovernance", new String[] {"i1", "i2"}); + parameterMap.put("disciPLINE", new String[] {"d1", "d2"}); + when(webRequest.getParameterMap()).thenReturn(parameterMap); + + InstitutionSearchRequest searchRequest = + (InstitutionSearchRequest) + instResolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertEquals(Arrays.asList("t1", "t2"), searchRequest.getType()); + assertEquals(Arrays.asList("i1", "i2"), searchRequest.getInstitutionalGovernance()); + assertEquals(Arrays.asList("d1", "d2"), searchRequest.getDisciplines()); + } + + @Test + public void collectionDescriptorsParamsTest() { + Map parameterMap = new HashMap<>(); + parameterMap.put("usageKey", new String[] {"1", "2"}); + parameterMap.put("usageName", new String[] {"n1", "n2"}); + parameterMap.put("usageRank", new String[] {"KINGDOM", "PHYLUM"}); + parameterMap.put("taxonKey", new String[] {"11", "22"}); + parameterMap.put("identifiedBy", new String[] {"ide1", "ide2"}); + parameterMap.put("typeStatus", new String[] {"t1", "t2"}); + parameterMap.put("recordedBy", new String[] {"rec1", "rec2"}); + parameterMap.put("discipline", new String[] {"d1", "d2"}); + parameterMap.put("objectClassification", new String[] {"oc1", "oc2"}); + parameterMap.put("issue", new String[] {"iss1", "iss2"}); + parameterMap.put("individualCount", new String[] {"111", "222"}); + parameterMap.put("descriptorCountry", new String[] {"DK", "ES"}); + parameterMap.put( + "dateIdentified", new String[] {"2024-01-01", "2024-01-01,*", "2024-01-01,2024-03-01"}); + when(webRequest.getParameterMap()).thenReturn(parameterMap); + + CollectionDescriptorsSearchRequestHandlerMethodArgumentResolver resolver = + new CollectionDescriptorsSearchRequestHandlerMethodArgumentResolver(); + CollectionDescriptorsSearchRequest searchRequest = + (CollectionDescriptorsSearchRequest) + resolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + + assertEquals(Arrays.asList(1, 2), searchRequest.getUsageKey()); + assertEquals(Arrays.asList("n1", "n2"), searchRequest.getUsageName()); + assertEquals(Arrays.asList(Rank.KINGDOM, Rank.PHYLUM), searchRequest.getUsageRank()); + assertEquals(Arrays.asList(11, 22), searchRequest.getTaxonKey()); + assertEquals(Arrays.asList("ide1", "ide2"), searchRequest.getIdentifiedBy()); + assertEquals(Arrays.asList("t1", "t2"), searchRequest.getTypeStatus()); + assertEquals(Arrays.asList("rec1", "rec2"), searchRequest.getRecordedBy()); + assertEquals(Arrays.asList("d1", "d2"), searchRequest.getDiscipline()); + assertEquals(Arrays.asList("oc1", "oc2"), searchRequest.getObjectClassification()); + assertEquals(Arrays.asList("iss1", "iss2"), searchRequest.getIssue()); + assertEquals(Arrays.asList("111", "222"), searchRequest.getIndividualCount()); + assertEquals( + Arrays.asList(Country.DENMARK, Country.SPAIN), searchRequest.getDescriptorCountry()); + assertEquals( + Arrays.asList("2024-01-01", "2024-01-01,*", "2024-01-01,2024-03-01"), + searchRequest.getDateIdentified()); + } + + @Test + public void facetsTest() { + mockFacetParams(); + InstitutionFacetedSearchRequestHandlerMethodArgumentResolver resolver = + new InstitutionFacetedSearchRequestHandlerMethodArgumentResolver(); + InstitutionFacetedSearchRequest searchRequest = + (InstitutionFacetedSearchRequest) + resolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertFacetParams(searchRequest, InstitutionFacetParameter.COUNTRY); + + mockFacetParams(); + CollectionDescriptorsSearchRequestHandlerMethodArgumentResolver collResolver = + new CollectionDescriptorsSearchRequestHandlerMethodArgumentResolver(); + CollectionDescriptorsSearchRequest collSearchRequest = + (CollectionDescriptorsSearchRequest) + collResolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertFacetParams(collSearchRequest, CollectionFacetParameter.COUNTRY); + } + + private static void assertFacetParams( + FacetedSearchRequest searchRequest, T facetParam) { + assertEquals(1, searchRequest.getFacets().size()); + assertEquals(facetParam, searchRequest.getFacets().iterator().next()); + assertTrue(searchRequest.isMultiSelectFacets()); + assertEquals(2, searchRequest.getFacetMinCount()); + assertEquals(1, searchRequest.getFacetLimit()); + assertEquals(2, searchRequest.getFacetOffset()); + assertEquals(1, searchRequest.getFacetPages().size()); + assertEquals(4, searchRequest.getFacetPages().get(facetParam).getOffset()); + } + + private void mockFacetParams() { + Map parameterMap = new HashMap<>(); + parameterMap.put("facet", new String[] {"COUNTRY"}); + parameterMap.put("facetMultiselect", new String[] {"true"}); + parameterMap.put("facetMincount", new String[] {"2"}); + parameterMap.put("facetLimit", new String[] {"1"}); + parameterMap.put("facetOffset", new String[] {"2"}); + parameterMap.put("country.facetOffset", new String[] {"4"}); + when(webRequest.getParameterMap()).thenReturn(parameterMap); + } + + @Test + public void collectionParamsTest() { + Map parameterMap = new HashMap<>(); + parameterMap.put( + "institution", + new String[] { + "b600ae47-97a9-4b63-85f3-21d4f52523a1", "d2434ba9-10ef-4308-bd45-a69af379b5d9" + }); + parameterMap.put("contentType", new String[] {"c1", "c2"}); + parameterMap.put("preservationType", new String[] {"p1", "p2"}); + parameterMap.put("accessionStatus", new String[] {"a1", "a2"}); + parameterMap.put("personalCollection", new String[] {"true", "false"}); + when(webRequest.getParameterMap()).thenReturn(parameterMap); + + CollectionSearchRequest collectionSearchRequest = + (CollectionSearchRequest) + collResolver.resolveArgument(methodParameter, mavContainer, webRequest, binderFactory); + assertEquals( + Arrays.asList( + UUID.fromString("b600ae47-97a9-4b63-85f3-21d4f52523a1"), + UUID.fromString("d2434ba9-10ef-4308-bd45-a69af379b5d9")), + collectionSearchRequest.getInstitution()); + assertEquals(Arrays.asList("c1", "c2"), collectionSearchRequest.getContentTypes()); + assertEquals(Arrays.asList("p1", "p2"), collectionSearchRequest.getPreservationTypes()); + assertEquals(Arrays.asList("a1", "a2"), collectionSearchRequest.getAccessionStatus()); + assertEquals(Arrays.asList(true, false), collectionSearchRequest.getPersonalCollection()); + } + + private void mockCommonParams() { + Map parameterMap = new HashMap<>(); + parameterMap.put("limit", new String[] {"200"}); + parameterMap.put("offset", new String[] {"10"}); + parameterMap.put("alternativeCode", new String[] {"ac1", "ac2"}); + parameterMap.put("CODE", new String[] {"c1"}); + parameterMap.put("name", new String[] {"n1"}); + parameterMap.put("contact", new String[] {"b600ae47-97a9-4b63-85f3-21d4f52523a1"}); + parameterMap.put("identifier", new String[] {"ide1", "ide2"}); + parameterMap.put( + "identifierType", + new String[] {IdentifierType.CITES.name(), IdentifierType.GRSCICOLL_URI.name()}); + parameterMap.put("machineTagName", new String[] {"mt1", "mt2"}); + parameterMap.put("machineTagNamespace", new String[] {"mt1", "mt2"}); + parameterMap.put("machineTagValue", new String[] {"mt1", "mt2"}); + parameterMap.put("city", new String[] {"city1"}); + parameterMap.put("fuzzyName", new String[] {"fuzzn1"}); + parameterMap.put("active", new String[] {"true"}); + parameterMap.put("masterSourceType", new String[] {MasterSourceType.IH.name()}); + parameterMap.put("numberSpecimens", new String[] {"1,*", "2,4", "3"}); + parameterMap.put("displayOnNHCPortal", new String[] {"true", "false"}); + parameterMap.put("occurrenceCount", new String[] {"1,*", "2,4", "3"}); + parameterMap.put("typeSpecimenCount", new String[] {"1,*", "2,4", "3"}); + parameterMap.put( + "institutionKey", + new String[] { + "b600ae47-97a9-4b63-85f3-21d4f52523a1", "d2434ba9-10ef-4308-bd45-a69af379b5d9" + }); + parameterMap.put("source", new String[] {Source.IH_IRN.name()}); + parameterMap.put("sourceId", new String[] {"214"}); + parameterMap.put("country", new String[] {"DK", "ES"}); + parameterMap.put("gbifRegion", new String[] {GbifRegion.EUROPE.name()}); + + when(webRequest.getParameterMap()).thenReturn(parameterMap); + when(webRequest.getParameter("hl")).thenReturn("true"); + when(webRequest.getParameter("q")).thenReturn("query"); + when(webRequest.getParameter("sortBy")) + .thenReturn(CollectionsSortField.NUMBER_SPECIMENS.name()); + when(webRequest.getParameter("sortOrder")).thenReturn(SortOrder.ASC.name()); + } + + private void assertCommonParams(SearchRequest searchRequest) { + assertEquals("query", searchRequest.getQ()); + assertEquals(200, searchRequest.getLimit()); + assertEquals(10, searchRequest.getOffset()); + assertEquals(CollectionsSortField.NUMBER_SPECIMENS, searchRequest.getSortBy()); + assertEquals(SortOrder.ASC, searchRequest.getSortOrder()); + assertEquals(Boolean.TRUE, searchRequest.getHl()); + assertEquals(Arrays.asList("ac1", "ac2"), searchRequest.getAlternativeCode()); + assertEquals(Collections.singletonList("c1"), searchRequest.getCode()); + assertEquals(Collections.singletonList("n1"), searchRequest.getName()); + assertEquals( + Collections.singletonList(UUID.fromString("b600ae47-97a9-4b63-85f3-21d4f52523a1")), + searchRequest.getContact()); + assertEquals(Arrays.asList("ide1", "ide2"), searchRequest.getIdentifier()); + assertEquals( + Arrays.asList(IdentifierType.CITES, IdentifierType.GRSCICOLL_URI), + searchRequest.getIdentifierType()); + assertEquals(Arrays.asList("mt1", "mt2"), searchRequest.getMachineTagName()); + assertEquals(Arrays.asList("mt1", "mt2"), searchRequest.getMachineTagNamespace()); + assertEquals(Arrays.asList("mt1", "mt2"), searchRequest.getMachineTagValue()); + assertEquals(Collections.singletonList("city1"), searchRequest.getCity()); + assertEquals(Collections.singletonList("fuzzn1"), searchRequest.getFuzzyName()); + assertEquals(Collections.singletonList(true), searchRequest.getActive()); + assertEquals( + Collections.singletonList(MasterSourceType.IH), searchRequest.getMasterSourceType()); + assertEquals(Arrays.asList("1,*", "2,4", "3"), searchRequest.getNumberSpecimens()); + assertEquals(Arrays.asList(true, false), searchRequest.getDisplayOnNHCPortal()); + assertEquals(Arrays.asList("1,*", "2,4", "3"), searchRequest.getOccurrenceCount()); + assertEquals(Arrays.asList("1,*", "2,4", "3"), searchRequest.getTypeSpecimenCount()); + assertEquals( + Arrays.asList( + UUID.fromString("b600ae47-97a9-4b63-85f3-21d4f52523a1"), + UUID.fromString("d2434ba9-10ef-4308-bd45-a69af379b5d9")), + searchRequest.getInstitutionKeys()); + assertEquals(Collections.singletonList(Source.IH_IRN), searchRequest.getSource()); + assertEquals(Collections.singletonList("214"), searchRequest.getSourceId()); + assertEquals(Arrays.asList(Country.DENMARK, Country.SPAIN), searchRequest.getCountry()); + assertEquals(Collections.singletonList(GbifRegion.EUROPE), searchRequest.getGbifRegion()); + } +} diff --git a/registry-integration-tests/src/test/resources/application-test.yml b/registry-integration-tests/src/test/resources/application-test.yml index 2591fa2065..16a58e9c6f 100644 --- a/registry-integration-tests/src/test/resources/application-test.yml +++ b/registry-integration-tests/src/test/resources/application-test.yml @@ -128,19 +128,19 @@ oaipmh: baseUrl: http://api.gbif-dev.org/v1/oai-pmh/registry adminEmail: gbifregistry@mailinator.com -dbmigration: - liquibase: - default-schema: public - change-log: classpath:liquibase/master.xml - enabled: true - datasource: - url: @registry-it.db.url@ - username: @registry-it.db.username@ - password: @registry-it.db.password@ - hikari: - idleTimeout: 60000 - maximumPoolSize: 1 - minimumIdle: 1 +#dbmigration: +# liquibase: +# default-schema: public +# change-log: classpath:liquibase/master.xml +# enabled: true +# datasource: +# url: @registry-it.db.url@ +# username: @registry-it.db.username@ +# password: @registry-it.db.password@ +# hikari: +# idleTimeout: 60000 +# maximumPoolSize: 1 +# minimumIdle: 1 spring: main: diff --git a/registry-mail/pom.xml b/registry-mail/pom.xml index 49ea23919b..0e6e78ccf8 100644 --- a/registry-mail/pom.xml +++ b/registry-mail/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-messaging/pom.xml b/registry-messaging/pom.xml index b17846cc0a..9e30bf9bea 100644 --- a/registry-messaging/pom.xml +++ b/registry-messaging/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-oaipmh/pom.xml b/registry-oaipmh/pom.xml index f5437d8e9b..4fcdd115a2 100644 --- a/registry-oaipmh/pom.xml +++ b/registry-oaipmh/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-persistence/pom.xml b/registry-persistence/pom.xml index cc2863af3a..9f282b0b63 100644 --- a/registry-persistence/pom.xml +++ b/registry-persistence/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java index 69a4e13021..fd8966420b 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/config/MyBatisConfiguration.java @@ -47,26 +47,8 @@ import org.gbif.registry.persistence.mapper.collections.external.IDigBioCollectionDto; import org.gbif.registry.persistence.mapper.collections.external.IdentifierDto; import org.gbif.registry.persistence.mapper.collections.external.MachineTagDto; -import org.gbif.registry.persistence.mapper.handler.*; import org.gbif.registry.persistence.mapper.dto.OrganizationGeoJsonDto; import org.gbif.registry.persistence.mapper.handler.*; -import org.gbif.registry.persistence.mapper.handler.CollectionContentTypeArrayTypeHandler; -import org.gbif.registry.persistence.mapper.handler.CountryNotNullTypeHandler; -import org.gbif.registry.persistence.mapper.handler.DOITypeHandler; -import org.gbif.registry.persistence.mapper.handler.DisciplineArrayTypeHandler; -import org.gbif.registry.persistence.mapper.handler.ExtensionArrayTypeHandler; -import org.gbif.registry.persistence.mapper.handler.InstitutionGovernanceArrayTypeHandler; -import org.gbif.registry.persistence.mapper.handler.LocaleTypeHandler; -import org.gbif.registry.persistence.mapper.handler.MetricInfoTypeHandler; -import org.gbif.registry.persistence.mapper.handler.OccurrenceDownloadStatusTypeHandler; -import org.gbif.registry.persistence.mapper.handler.PredicateTypeHandler; -import org.gbif.registry.persistence.mapper.handler.StepTypeArrayTypeHandler; -import org.gbif.registry.persistence.mapper.handler.SuggestedChangesTypeHandler; -import org.gbif.registry.persistence.mapper.handler.UserIdsTypeHandler; - -import java.net.URI; -import java.util.UUID; - import org.mybatis.spring.boot.autoconfigure.ConfigurationCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/CollectionSearchDto.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/CollectionSearchDto.java index 285242b4ba..b67d301b62 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/CollectionSearchDto.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/CollectionSearchDto.java @@ -6,7 +6,6 @@ import lombok.EqualsAndHashCode; import org.gbif.api.vocabulary.collections.MasterSourceType; -@SuppressWarnings("MissingOverride") @EqualsAndHashCode(callSuper = true) @Data public class CollectionSearchDto extends SearchDto { diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/InstitutionSearchDto.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/InstitutionSearchDto.java index 32f7cf2c3f..403ab7a6c7 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/InstitutionSearchDto.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/InstitutionSearchDto.java @@ -12,7 +12,6 @@ import org.gbif.api.vocabulary.License; import org.gbif.api.vocabulary.collections.MasterSourceType; -@SuppressWarnings("MissingOverride") @EqualsAndHashCode(callSuper = true) @Data public class InstitutionSearchDto extends BaseSearchDto { diff --git a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/SearchDto.java b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/SearchDto.java index 4827400df0..fb5265313b 100644 --- a/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/SearchDto.java +++ b/registry-persistence/src/main/java/org/gbif/registry/persistence/mapper/collections/dto/SearchDto.java @@ -21,7 +21,6 @@ import org.gbif.api.vocabulary.Country; import org.gbif.api.vocabulary.Rank; -@SuppressWarnings("MissingOverride") @EqualsAndHashCode(callSuper = true) @Data public class SearchDto extends BaseSearchDto { diff --git a/registry-persistence/src/main/resources/liquibase/143-removed-fields-grscicoll.xml b/registry-persistence/src/main/resources/liquibase/143-removed-fields-grscicoll.xml index d96c874ef7..fdc3254605 100644 --- a/registry-persistence/src/main/resources/liquibase/143-removed-fields-grscicoll.xml +++ b/registry-persistence/src/main/resources/liquibase/143-removed-fields-grscicoll.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd"> - + + + + + + + + diff --git a/registry-persistence/src/main/resources/liquibase/master.xml b/registry-persistence/src/main/resources/liquibase/master.xml index 1f89f9ba5a..0188927f99 100644 --- a/registry-persistence/src/main/resources/liquibase/master.xml +++ b/registry-persistence/src/main/resources/liquibase/master.xml @@ -157,4 +157,5 @@ + diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/ContactMapper.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/ContactMapper.xml index 91bc91075f..d2f2cc2053 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/ContactMapper.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/ContactMapper.xml @@ -13,7 +13,7 @@ - user_id,first_name,last_name,description,position,email,phone,homepage,organization,address,city,province,country,postal_code,created,created_by,modified,modified_by + user_id,first_name,last_name,salutation,description,position,email,phone,homepage,organization,address,city,province,country,postal_code,created,created_by,modified,modified_by @@ -24,6 +24,7 @@ #{userId, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, #{firstName, jdbcType=VARCHAR}, #{lastName, jdbcType=VARCHAR}, + #{salutation, jdbcType=VARCHAR}, #{description, jdbcType=VARCHAR}, #{position, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, #{email, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, @@ -45,6 +46,7 @@ user_id = #{userId, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, first_name = #{firstName, jdbcType=VARCHAR}, last_name = #{lastName, jdbcType=VARCHAR}, + salutation = #{salutation, jdbcType=VARCHAR}, description = #{description, jdbcType=VARCHAR}, position = #{position, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, email = #{email, jdbcType=ARRAY, typeHandler=org.gbif.mybatis.type.StringArrayTypeHandler}, diff --git a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/Common.xml b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/Common.xml index 0ade1e7193..b5e4cd1d74 100644 --- a/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/Common.xml +++ b/registry-persistence/src/main/resources/org/gbif/registry/persistence/mapper/collections/Common.xml @@ -143,10 +143,10 @@ #{item,jdbcType=VARCHAR} - OR mail_addr.country IN + OR (addr.country IS NULL AND mail_addr.country IN #{item,jdbcType=VARCHAR} - + ) ) @@ -154,10 +154,10 @@ #{item,jdbcType=VARCHAR} - OR mail_addr.country IN + OR (addr.country IS NULL AND mail_addr.country IN #{item,jdbcType=VARCHAR} - + ) ) @@ -165,7 +165,7 @@ normalize_name(addr.city) = normalize_name(#{item,jdbcType=VARCHAR}) OR - normalize_name(mail_addr.city) = normalize_name(#{item,jdbcType=VARCHAR}) OR + (addr.city IS NULL AND normalize_name(mail_addr.city) = normalize_name(#{item,jdbcType=VARCHAR})) OR ) diff --git a/registry-pipelines/pom.xml b/registry-pipelines/pom.xml index a47aa07f03..63461c0649 100644 --- a/registry-pipelines/pom.xml +++ b/registry-pipelines/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 @@ -51,14 +51,6 @@ org.gbif.occurrence occurrence-ws-client - - com.squareup.okhttp3 - okhttp - - - com.squareup.okio - okio - diff --git a/registry-pipelines/src/main/java/org/gbif/registry/pipelines/DefaultRegistryPipelinesHistoryTrackingService.java b/registry-pipelines/src/main/java/org/gbif/registry/pipelines/DefaultRegistryPipelinesHistoryTrackingService.java index 46a59c9d57..91ffdbd21e 100644 --- a/registry-pipelines/src/main/java/org/gbif/registry/pipelines/DefaultRegistryPipelinesHistoryTrackingService.java +++ b/registry-pipelines/src/main/java/org/gbif/registry/pipelines/DefaultRegistryPipelinesHistoryTrackingService.java @@ -49,7 +49,6 @@ import java.io.IOException; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; @@ -65,6 +64,8 @@ import java.util.concurrent.Executors; import java.util.function.Consumer; +import com.fasterxml.jackson.core.JsonProcessingException; + import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; @@ -102,18 +103,8 @@ public class DefaultRegistryPipelinesHistoryTrackingService EndpointCreatedComparator.INSTANCE)); private static final Set FINISHED_STATE_SET = - new HashSet<>( - Arrays.asList( - PipelineStep.Status.COMPLETED, - PipelineStep.Status.ABORTED, - PipelineStep.Status.FAILED)); - - private static final Set INCLUDE_FRAGMENTER = new HashSet<>(Arrays.asList( - StepType.DWCA_TO_VERBATIM, - StepType.XML_TO_VERBATIM, - StepType.ABCD_TO_VERBATIM, - StepType.FRAGMENTER - )); + Set.of( + PipelineStep.Status.COMPLETED, PipelineStep.Status.ABORTED, PipelineStep.Status.FAILED); private ObjectMapper objectMapper; /** The messagePublisher can be optional. */ @@ -396,9 +387,12 @@ public RunPipelineResponse runPipelineAttempt( // Performs the messaging and updates the status onces the message has been sent Dataset dataset = datasetService.get(datasetKey); + // Create Rabbit messages Map stepsToSend = new EnumMap<>(StepType.class); for (StepType stepName : prioritizeSteps(steps, dataset)) { - processStep(stepName, process, prefix, interpretTypes, stepsToSend); + Optional message = + createStepMessage(stepName, process, prefix, interpretTypes); + message.ifPresent(m -> stepsToSend.put(stepName, m)); } if (stepsToSend.isEmpty()) { @@ -452,66 +446,78 @@ public RunPipelineResponse runPipelineAttempt( return responseBuilder.build(); } - private void processStep(StepType stepName, PipelineProcess process, String prefix, - Set interpretTypes, Map stepsToSend){ + private Optional createStepMessage(StepType stepName, PipelineProcess process, String prefix, + Set interpretTypes){ Optional latestStepOpt = getLatestSuccessfulStep(process, stepName); - if (!latestStepOpt.isPresent()) { + if (latestStepOpt.isEmpty()) { LOG.warn("Can't find latest successful step for the datasetKey {}", process.getDatasetKey()); - return; + return Optional.empty(); } - PipelineStep step = latestStepOpt.get(); - try { - PipelineBasedMessage message = null; - - if (stepName == StepType.INTERPRETED_TO_INDEX - || stepName == StepType.HDFS_VIEW - || stepName == StepType.FRAGMENTER) { - message = createInterpretedMessage(prefix, step.getMessage(), stepName); - } else if (stepName == StepType.VERBATIM_TO_INTERPRETED) { - message = createVerbatimMessage(prefix, step.getMessage(), interpretTypes); - } else if (stepName == StepType.DWCA_TO_VERBATIM) { - message = createMessage(step.getMessage(), PipelinesDwcaMessage.class); - } else if (stepName == StepType.ABCD_TO_VERBATIM) { - message = createMessage(step.getMessage(), PipelinesAbcdMessage.class); - } else if (stepName == StepType.XML_TO_VERBATIM) { - message = createMessage(step.getMessage(), PipelinesXmlMessage.class); - } else if (stepName == StepType.EVENTS_VERBATIM_TO_INTERPRETED) { - message = createMessage(step.getMessage(), PipelinesEventsMessage.class); - } else if (stepName == StepType.EVENTS_INTERPRETED_TO_INDEX) { - message = createMessage(step.getMessage(), PipelinesEventsInterpretedMessage.class); - } - - if (message != null) { - stepsToSend.put(stepName, message); - } - } catch (IOException ex) { - LOG.warn("Error reading message", ex); + String jsonMessage = latestStepOpt.get().getMessage(); + switch (stepName) { + case INTERPRETED_TO_INDEX: + case HDFS_VIEW: + case FRAGMENTER: + return createInterpretedMessage(prefix, jsonMessage, stepName); + case VERBATIM_TO_INTERPRETED: + return createVerbatimMessage(prefix, jsonMessage, interpretTypes); + case DWCA_TO_VERBATIM: + return deserializeMessage(jsonMessage, PipelinesDwcaMessage.class); + case ABCD_TO_VERBATIM: + return deserializeMessage(jsonMessage, PipelinesAbcdMessage.class); + case XML_TO_VERBATIM: + return deserializeMessage(jsonMessage, PipelinesXmlMessage.class); + case EVENTS_VERBATIM_TO_INTERPRETED: + return deserializeMessage(jsonMessage, PipelinesEventsMessage.class); + case EVENTS_INTERPRETED_TO_INDEX: + return deserializeMessage(jsonMessage, PipelinesEventsInterpretedMessage.class); + default: + return Optional.empty(); } } @VisibleForTesting protected Set getStepTypes(Set stepsToSend) { - Set finalSteps = - PipelinesWorkflow.getOccurrenceWorkflow().getAllNodesFor(stepsToSend); - if (stepsToSend.stream().noneMatch(INCLUDE_FRAGMENTER::contains)) { + Set finalSteps = new HashSet<>(); + if (stepsToSend.stream().anyMatch(StepType::isEventType)) { + finalSteps.addAll(PipelinesWorkflow.getEventWorkflow().getAllNodesFor(stepsToSend)); + } + if (stepsToSend.stream().anyMatch(StepType::isOccurrenceType)) { + finalSteps.addAll(PipelinesWorkflow.getOccurrenceWorkflow().getAllNodesFor(stepsToSend)); + } + // TODO: Events from dwca will be ignorred, add dataset type? + if (stepsToSend.stream().anyMatch(StepType::isVerbatimType)) { + finalSteps.addAll(PipelinesWorkflow.getOccurrenceWorkflow().getAllNodesFor(stepsToSend)); + } + if (stepsToSend.stream().noneMatch(StepType::isVerbatimType)) { finalSteps.remove(StepType.FRAGMENTER); } return finalSteps; } - private T createMessage(String jsonMessage, Class targetClass) - throws IOException { - return objectMapper.readValue(jsonMessage, targetClass); + private Optional deserializeMessage( + String jsonMessage, Class targetClass) { + try { + T message = objectMapper.readValue(jsonMessage, targetClass); + return Optional.of(message); + } catch (JsonProcessingException ex) { + LOG.warn("Error deserializing json message", ex); + return Optional.empty(); + } } - private PipelineBasedMessage createVerbatimMessage( - String prefix, String jsonMessage, Set interpretTypes) throws IOException { + private Optional createVerbatimMessage( + String prefix, String jsonMessage, Set interpretTypes) { PipelinesVerbatimMessage message = - objectMapper.readValue(jsonMessage, PipelinesVerbatimMessage.class); + deserializeMessage(jsonMessage, PipelinesVerbatimMessage.class).orElse(null); + if (message == null) { + return Optional.empty(); + } + Optional.ofNullable(prefix).ifPresent(message::setResetPrefix); - HashSet steps = new HashSet<>(); + Set steps = new HashSet<>(); if (message.getPipelineSteps().contains(StepType.VERBATIM_TO_INTERPRETED.name())) { steps.add(StepType.VERBATIM_TO_INTERPRETED.name()); @@ -539,16 +545,19 @@ private PipelineBasedMessage createVerbatimMessage( message.getInterpretTypes().add(RecordType.CLUSTERING.name()); } - return message; + return Optional.of(message); } - private PipelineBasedMessage createInterpretedMessage( - String prefix, String jsonMessage, StepType stepType) throws IOException { + private Optional createInterpretedMessage( + String prefix, String jsonMessage, StepType stepType) { PipelinesInterpretedMessage message = - objectMapper.readValue(jsonMessage, PipelinesInterpretedMessage.class); + deserializeMessage(jsonMessage, PipelinesInterpretedMessage.class).orElse(null); + if (message == null) { + return Optional.empty(); + } Optional.ofNullable(prefix).ifPresent(message::setResetPrefix); - message.setPipelineSteps(Collections.singleton(stepType.name())); - return message; + message.setPipelineSteps(Set.of(stepType.name())); + return Optional.of(message); } @Override diff --git a/registry-pipelines/src/main/java/org/gbif/registry/pipelines/issues/BasicAuthInterceptor.java b/registry-pipelines/src/main/java/org/gbif/registry/pipelines/issues/BasicAuthInterceptor.java deleted file mode 100644 index 31ba353540..0000000000 --- a/registry-pipelines/src/main/java/org/gbif/registry/pipelines/issues/BasicAuthInterceptor.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.gbif.registry.pipelines.issues; - -import java.io.IOException; - -import okhttp3.CacheControl; -import okhttp3.Credentials; -import okhttp3.Interceptor; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; - -/** Interceptor for the {@link OkHttpClient} to add basic auth in all the requests. */ -public class BasicAuthInterceptor implements Interceptor { - - private String credentials; - - public BasicAuthInterceptor(String user, String password) { - this.credentials = Credentials.basic(user, password); - } - - @Override - public Response intercept(Chain chain) throws IOException { - Request request = chain.request(); - Request authenticatedRequest = - request - .newBuilder() - .header("Authorization", credentials) - .cacheControl(new CacheControl.Builder().noCache().build()) - .build(); - return chain.proceed(authenticatedRequest); - } -} diff --git a/registry-pipelines/src/test/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingServiceTest.java b/registry-pipelines/src/test/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingServiceTest.java index 561a15ba33..5a522de75d 100644 --- a/registry-pipelines/src/test/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingServiceTest.java +++ b/registry-pipelines/src/test/java/org/gbif/registry/pipelines/RegistryPipelinesHistoryTrackingServiceTest.java @@ -60,7 +60,7 @@ void getLatestSuccesfulStepTest() { execution.addStep(s1); execution.addStep(s2); execution.addStep(s3); - execution.setStepsToRun(Collections.singleton(StepType.ABCD_TO_VERBATIM)); + execution.setStepsToRun(Set.of(StepType.ABCD_TO_VERBATIM)); PipelineProcess process = new PipelineProcess(); process.addExecution(execution); @@ -73,7 +73,7 @@ void getLatestSuccesfulStepTest() { @Test void getStepTypesFragmenterTest() { Set result = - trackingService.getStepTypes(Collections.singleton(StepType.ABCD_TO_VERBATIM)); + trackingService.getStepTypes(Set.of(StepType.ABCD_TO_VERBATIM)); assertTrue(result.contains(StepType.FRAGMENTER)); } @@ -82,8 +82,20 @@ void getStepTypesFragmenterTest() { void getStepTypesTest() { Set result = - trackingService.getStepTypes(Collections.singleton(StepType.VERBATIM_TO_INTERPRETED)); + trackingService.getStepTypes(Set.of(StepType.VERBATIM_TO_INTERPRETED)); assertFalse(result.contains(StepType.FRAGMENTER)); } + + @Test + void getStepTypesEventTest() { + + Set result = + trackingService.getStepTypes(Set.of(StepType.EVENTS_VERBATIM_TO_INTERPRETED)); + + assertTrue(result.contains(StepType.EVENTS_VERBATIM_TO_INTERPRETED)); + assertTrue(result.contains(StepType.EVENTS_HDFS_VIEW)); + assertTrue(result.contains(StepType.EVENTS_INTERPRETED_TO_INDEX)); + assertFalse(result.contains(StepType.FRAGMENTER)); + } } diff --git a/registry-search/pom.xml b/registry-search/pom.xml index c4b5a00d8d..ac6d6483c2 100644 --- a/registry-search/pom.xml +++ b/registry-search/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-security/pom.xml b/registry-security/pom.xml index d00a938cf0..60c30e163e 100644 --- a/registry-security/pom.xml +++ b/registry-security/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-service/pom.xml b/registry-service/pom.xml index 0c4fb45f35..ca4c5a0afb 100644 --- a/registry-service/pom.xml +++ b/registry-service/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java b/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java index 0363213b64..afcead2432 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/CollectionsSearchService.java @@ -392,7 +392,7 @@ private void buildCommonParams( ListParams.ListParamsBuilder listParams, SearchRequest searchRequest) { String query = searchRequest.getQ() != null - ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(searchRequest.getQ())) + ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(searchRequest.getQ())) : searchRequest.getQ(); listParams diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultCollectionService.java b/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultCollectionService.java index f0e3d83d84..52cc2a26e0 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultCollectionService.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultCollectionService.java @@ -150,7 +150,7 @@ private PagingResponse listInternal( String query = searchRequest.getQ() != null - ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(searchRequest.getQ())) + ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(searchRequest.getQ())) : searchRequest.getQ(); Set institutionKeys = new HashSet<>(); diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultInstitutionService.java b/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultInstitutionService.java index be445dbcaa..98a5950ac0 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultInstitutionService.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/DefaultInstitutionService.java @@ -186,7 +186,7 @@ private InstitutionListParams buildSearchParams( InstitutionSearchRequest searchRequest, boolean deleted, Pageable page) { String query = searchRequest.getQ() != null - ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(searchRequest.getQ())) + ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(searchRequest.getQ())) : searchRequest.getQ(); return InstitutionListParams.builder() diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/DefaultDescriptorService.java b/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/DefaultDescriptorService.java index 842a39f2d7..7bb9537589 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/DefaultDescriptorService.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/DefaultDescriptorService.java @@ -294,7 +294,7 @@ public PagingResponse listDescriptorGroups( Pageable page = searchRequest.getPage() == null ? new PagingRequest() : searchRequest.getPage(); String query = searchRequest.getQ() != null - ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(searchRequest.getQ())) + ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(searchRequest.getQ())) : searchRequest.getQ(); DescriptorGroupParams params = @@ -337,7 +337,7 @@ private DescriptorParams createDescriptorParams(DescriptorSearchRequest searchRe Pageable page = searchRequest.getPage() == null ? new PagingRequest() : searchRequest.getPage(); String query = searchRequest.getQ() != null - ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(searchRequest.getQ())) + ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(searchRequest.getQ())) : searchRequest.getQ(); return DescriptorParams.builder() diff --git a/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/Interpreter.java b/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/Interpreter.java index 4912b6eb79..49eae7cb7c 100644 --- a/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/Interpreter.java +++ b/registry-service/src/main/java/org/gbif/registry/service/collections/descriptors/Interpreter.java @@ -19,6 +19,7 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.Singular; import org.gbif.api.model.checklistbank.NameUsage; import org.gbif.api.model.checklistbank.NameUsageMatch; import org.gbif.api.v2.NameUsageMatch2; @@ -46,7 +47,7 @@ public class Interpreter { private static final MultiinputTemporalParser temporalParser = MultiinputTemporalParser.create(); private static final CountryParser countryParser = CountryParser.getInstance(); private static final Set SUSPECTED_TYPE_STATUS_VALUES = - new HashSet<>(Arrays.asList("?", "possible", "possibly", "potential", "maybe", "perhaps")); + Set.of("?", "possible", "possibly", "potential", "maybe", "perhaps"); public static InterpretedResult> interpretStringList( Map valuesMap, DwcTerm term) { @@ -326,13 +327,11 @@ public static InterpretedResult interpretTaxonomy( } if (nameUsageMatch.getClassification() != null) { - List rankedNames = new ArrayList<>(); - taxonDataBuilder.taxonClassification(rankedNames); nameUsageMatch .getClassification() .forEach( c -> { - rankedNames.add(c); + taxonDataBuilder.rankedName(c); taxonKeys.add(c.getKey()); }); } @@ -388,6 +387,8 @@ static class TaxonData { private Integer usageKey; private String usageName; private Rank usageRank; + + @Singular("rankedName") private List taxonClassification; private Set taxonKeys; diff --git a/registry-surety/pom.xml b/registry-surety/pom.xml index 085d449208..5cf6d919ec 100644 --- a/registry-surety/pom.xml +++ b/registry-surety/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-ws-client/pom.xml b/registry-ws-client/pom.xml index 428d2ccd20..9176c76cbe 100644 --- a/registry-ws-client/pom.xml +++ b/registry-ws-client/pom.xml @@ -3,7 +3,7 @@ registry-parent org.gbif.registry - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT 4.0.0 diff --git a/registry-ws/pom.xml b/registry-ws/pom.xml index f8cd59a96b..2bb43d8720 100644 --- a/registry-ws/pom.xml +++ b/registry-ws/pom.xml @@ -4,7 +4,7 @@ org.gbif.registry registry-parent - 3.96.30-SNAPSHOT + 3.97.0-SNAPSHOT registry-ws GBIF Registry Webservices @@ -19,6 +19,20 @@ + + + org.apache.curator + curator-framework + + + org.apache.curator + curator-recipes + + + org.apache.curator + curator-x-discovery + + org.projectlombok diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/provider/CollectionSearchRequestHandlerMethodArgumentResolver.java b/registry-ws/src/main/java/org/gbif/registry/ws/provider/CollectionSearchRequestHandlerMethodArgumentResolver.java index 01bde48051..563fba3a9a 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/provider/CollectionSearchRequestHandlerMethodArgumentResolver.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/provider/CollectionSearchRequestHandlerMethodArgumentResolver.java @@ -54,7 +54,6 @@ protected void fillCollectionSearchRequest( extractMultivalueParam(params, "preservationType") .ifPresent(searchRequest::setPreservationTypes); extractMultivalueParam(params, "accessionStatus").ifPresent(searchRequest::setAccessionStatus); - extractMultivalueParam(params, "contentType").ifPresent(searchRequest::setContentTypes); extractMultivalueParam(params, "personalCollection", Boolean::parseBoolean) .ifPresent(searchRequest::setPersonalCollection); } diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/BaseNetworkEntityResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/BaseNetworkEntityResource.java index 64b12e28d2..246e78f961 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/BaseNetworkEntityResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/BaseNetworkEntityResource.java @@ -193,7 +193,7 @@ protected BaseNetworkEntityResource( @Parameter( name = "modified", description = - "The modified date of the dataset. Accepts ranges and a '*' can be used as a wildcard, e.g.:modified=2023-04-01,*", + "The modified date of the dataset. Accepts ranges and a `*` can be used as a wildcard, e.g. `modified=2023-04-01,*`", schema = @Schema(implementation = Range.class), in = ParameterIn.QUERY, explode = Explode.TRUE), @@ -281,7 +281,7 @@ public PagingResponse list(Pageable page) { @Override @Deprecated public PagingResponse search(String query, Pageable page) { - String q = query != null ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(query)) : query; + String q = query != null ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(query)) : query; return list(BaseListParams.builder().query(q).page(page).build()); } @@ -973,7 +973,7 @@ protected PagingResponse pagingResponse(Pageable page, Long count, List range) { diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java index b048cb67a6..6249fd8259 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/DatasetResource.java @@ -311,14 +311,14 @@ public DatasetResource( @Parameter( name = "recordCount", description = - "Number of records of the dataset. Accepts ranges and a '*' can be used as a wildcard.", + "Number of records of the dataset. Accepts ranges and a `*` can be used as a wildcard.", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY, example = "100,*"), @Parameter( name = "modifiedDate", description = - "Date when the dataset was modified the last time. Accepts ranges and a '*' can be used as a wildcard.", + "Date when the dataset was modified the last time. Accepts ranges and a `*` can be used as a wildcard.", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY, example = "2022-05-01,*"), diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OrganizationResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OrganizationResource.java index a226522c3e..8f7500ece5 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/OrganizationResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/OrganizationResource.java @@ -299,7 +299,7 @@ protected static String generatePassword() { @Override public PagingResponse search(String query, Pageable page) { - String q = query != null ? Strings.emptyToNull(CharMatcher.WHITESPACE.trimFrom(query)) : query; + String q = query != null ? Strings.emptyToNull(CharMatcher.whitespace().trimFrom(query)) : query; OrganizationListParams listParams = OrganizationListParams.builder().query(q).page(page).build(); long total = organizationMapper.count(listParams); diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/UserManagementResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/UserManagementResource.java index 52c41d954e..feb8647618 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/UserManagementResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/UserManagementResource.java @@ -304,7 +304,7 @@ public PagingResponse search( page = page == null ? new PagingRequest() : page; String q = Optional.ofNullable(query) - .map(v -> Strings.nullToEmpty(CharMatcher.WHITESPACE.trimFrom(v))) + .map(v -> Strings.nullToEmpty(CharMatcher.whitespace().trimFrom(v))) .orElse(null); Set countries = Optional.ofNullable(countryRightsOn) diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/BaseCollectionEntityResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/BaseCollectionEntityResource.java index 7592741b05..423d3a0d15 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/BaseCollectionEntityResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/BaseCollectionEntityResource.java @@ -210,7 +210,7 @@ protected BaseCollectionEntityResource( @Parameter( name = "numberSpecimens", description = - "Number of specimens. It supports ranges and a '*' can be used as a wildcard", + "Number of specimens. It supports ranges and a `*` can be used as a wildcard", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY), @Parameter( @@ -226,13 +226,13 @@ protected BaseCollectionEntityResource( @Parameter( name = "occurrenceCount", description = - "Count of occurrences linked. It supports ranges and a '*' can be used as a wildcard", + "Count of occurrences linked. It supports ranges and a `*` can be used as a wildcard", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY), @Parameter( name = "typeSpecimenCount", description = - "Count of type specimens linked. It supports ranges and a '*' can be used as a wildcard", + "Count of type specimens linked. It supports ranges and a `*` can be used as a wildcard", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY), @Parameter( diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/CollectionResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/CollectionResource.java index 9f7e3c3a36..71eadd9e12 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/CollectionResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/collections/CollectionResource.java @@ -42,7 +42,6 @@ import java.nio.file.attribute.PosixFilePermissions; import java.util.Arrays; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.UUID; @@ -716,14 +715,13 @@ public ResponseEntity exportDescriptorGroup( } private static Set filePermissions() { - return new HashSet<>( - Arrays.asList( - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.OTHERS_WRITE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE)); + return Set.of( + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_EXECUTE, + PosixFilePermission.OTHERS_WRITE, + PosixFilePermission.OTHERS_READ, + PosixFilePermission.OTHERS_EXECUTE); } private static Path zipFiles( @@ -800,7 +798,7 @@ private static Path zipFiles( @Parameter( name = "individualCount", description = - "Individual count of the descriptor. It supports ranges and a '*' can be used as a wildcard", + "Individual count of the descriptor. It supports ranges and a `*` can be used as a wildcard", schema = @Schema(implementation = String.class), in = ParameterIn.QUERY), @Parameter( diff --git a/registry-ws/src/main/java/org/gbif/registry/ws/resources/legacy/IptResource.java b/registry-ws/src/main/java/org/gbif/registry/ws/resources/legacy/IptResource.java index daf077b0de..267508db04 100644 --- a/registry-ws/src/main/java/org/gbif/registry/ws/resources/legacy/IptResource.java +++ b/registry-ws/src/main/java/org/gbif/registry/ws/resources/legacy/IptResource.java @@ -220,11 +220,13 @@ public ResponseEntity updateIpt( /** * Register IPT dataset, handling incoming request with path /ipt/resource. The primary contact * and publishing organization key are mandatory. Only after both the dataset and primary contact - * have been persisted is a response with {@link HttpStatus#CREATED} returned.
Before being + * have been persisted is a response with {@link HttpStatus#CREATED} returned. + *

+ * Before being * persisted, the dataset is the UNSPECIFIED license. This will be replaced (if possible) by the * publisher assigned license when the dataset gets crawled the first time. Since IPT 2.2, the IPT * EML metadata document always includes a machine-readable license. See discussion at - * https://github.com/gbif/registry/issues/71 + * GitHub * * @param dataset LegacyDataset with HTTP form parameters * @return {@link ResponseEntity} with {@link HttpStatus#CREATED} if successful