From 0db7cd96e100e0c896c040c895cd6a09f39e903a Mon Sep 17 00:00:00 2001 From: 51-code <146736881+51-code@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:26:09 +0300 Subject: [PATCH] JSON Array splitting (#77) * Allow splitting incoming messages with JSON array format Delete PayloadConfig and include configuration in ConversionFactory that provides the correct object instead Add secondary ctor to RegexConversion Use Collections.singletonList in DefaultPayload, RegexConversion as final class Add object equality test for RegexPayload Move tests to the correct package Move config tests from RegexPayloadTest to PayloadConfigTest and add tests for splitType Refactor split() function in Payload Refactor JsonPayload and RegexPayload to encapsulate Payload instead of String Rename Payload's take() function to message() Implement json_array splitting Refactor Payload objects and regex splitting Remove duplicate tests from RegexSplittingTest Apply spotless Move regex splitting tests to their own file, refactor the tests Change config to allow splitting with json_array Refactor regex splitting to a decorator * Fix rebase * Use try-with-resources when loading configuration file * Add object equality tests for ConversionFactory * Split object equality test into two tests in RegexPayloadTest * Added missing equals and hashCode functions, added equality tests * Add missing hashCodes to ConversionFactory and LookupConfig * Remove checking System Properties in ConversionFactory, make end-to-end tests use properties from a resource file * Use rlo_06 for asserting payload correctness, add multithreading test * Use expected message amounts instead of result list size as an expected value in assertions --- README.adoc | 4 +- etc/config.properties | 4 +- pom.xml | 19 ++ .../com/teragrep/lsh_01/HttpInitializer.java | 1 + .../teragrep/lsh_01/HttpServerHandler.java | 1 + src/main/java/com/teragrep/lsh_01/Main.java | 20 +- .../com/teragrep/lsh_01/MessageProcessor.java | 1 + .../com/teragrep/lsh_01/NettyHttpServer.java | 1 + .../java/com/teragrep/lsh_01/Payload.java | 64 ----- .../authentication/BasicAuthentication.java | 16 ++ .../teragrep/lsh_01/config/Configuration.java | 34 +++ .../lsh_01/config/ConversionFactory.java | 145 ++++++++++ .../teragrep/lsh_01/config/LookupConfig.java | 17 ++ .../lsh_01/config/PathProperties.java | 73 +++++ .../teragrep/lsh_01/config/PayloadConfig.java | 56 ---- .../lsh_01/config/SecurityConfig.java | 17 ++ .../lsh_01/conversion/DefaultPayload.java | 55 ++++ .../{ => conversion}/IMessageHandler.java | 2 +- .../lsh_01/conversion/JsonConversion.java | 70 +++++ .../lsh_01/conversion/JsonPayload.java | 76 ++++++ .../MetricRelpConversion.java | 2 +- .../teragrep/lsh_01/conversion/Payload.java | 35 +++ .../lsh_01/conversion/RegexConversion.java | 77 ++++++ .../lsh_01/conversion/RegexPayload.java | 73 +++++ .../{ => conversion}/RelpConversion.java | 29 +- .../java/com/teragrep/lsh_01/pool/Pool.java | 1 - .../lsh_01/pool/RelpConnectionFactory.java | 4 + .../lsh_01/BasicAuthenticationTest.java | 77 ++++++ .../lsh_01/ConversionFactoryTest.java | 252 ++++++++++++++++++ .../com/teragrep/lsh_01/CredentialsTest.java | 32 +-- .../teragrep/lsh_01/DefaultPayloadTest.java | 56 ++++ .../com/teragrep/lsh_01/EndToEndTest.java | 77 +----- .../com/teragrep/lsh_01/JsonPayloadTest.java | 94 +++++++ .../teragrep/lsh_01/JsonSplittingTest.java | 220 +++++++++++++++ .../java/com/teragrep/lsh_01/LookupTest.java | 11 +- .../java/com/teragrep/lsh_01/MetricTest.java | 6 +- .../lsh_01/MultithreadingEndToEndTest.java | 139 ++++++++++ .../teragrep/lsh_01/PathPropertiesTest.java | 57 ++++ .../java/com/teragrep/lsh_01/PayloadTest.java | 108 -------- .../com/teragrep/lsh_01/RegexPayloadTest.java | 111 ++++++++ .../teragrep/lsh_01/RegexSplittingTest.java | 153 +++++++++++ .../teragrep/lsh_01/SecurityConfigTest.java | 64 +++++ src/test/resources/exampleCredentials.json | 6 + .../properties/customTest.properties | 26 ++ .../properties/defaultTest.properties | 26 ++ .../properties/jsonSplittingTest.properties | 26 ++ .../properties/multithreadingTest.properties | 26 ++ .../properties/regexSplittingTest.properties | 26 ++ 48 files changed, 2132 insertions(+), 358 deletions(-) delete mode 100644 src/main/java/com/teragrep/lsh_01/Payload.java create mode 100644 src/main/java/com/teragrep/lsh_01/config/Configuration.java create mode 100644 src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java create mode 100644 src/main/java/com/teragrep/lsh_01/config/PathProperties.java delete mode 100644 src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java rename src/main/java/com/teragrep/lsh_01/{ => conversion}/IMessageHandler.java (97%) create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java rename src/main/java/com/teragrep/lsh_01/{ => conversion}/MetricRelpConversion.java (98%) create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/Payload.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java create mode 100644 src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java rename src/main/java/com/teragrep/lsh_01/{ => conversion}/RelpConversion.java (80%) create mode 100644 src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java delete mode 100644 src/test/java/com/teragrep/lsh_01/PayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java create mode 100644 src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java create mode 100644 src/test/resources/exampleCredentials.json create mode 100644 src/test/resources/properties/customTest.properties create mode 100644 src/test/resources/properties/defaultTest.properties create mode 100644 src/test/resources/properties/jsonSplittingTest.properties create mode 100644 src/test/resources/properties/multithreadingTest.properties create mode 100644 src/test/resources/properties/regexSplittingTest.properties diff --git a/README.adoc b/README.adoc index 710aa0c4..ca8f5174 100644 --- a/README.adoc +++ b/README.adoc @@ -35,8 +35,8 @@ security.authRequired,true,Sets whether Basic HTTP Authorization headers are req credentials.file,etc/credentials.json,A json file with array of identity:credential mappings lookups.hostname.file,etc/hostname.json,Path to username-to-hostname lookup table lookups.appname.file,etc/appname.json,Path to username-to-appname lookup table -payload.splitRegex, \n (newline), A regex based on which incoming requests will be split into multiple outgoing messages -payload.splitEnabled, false, Sets whether splitting incoming messages by splitRegex is enabled +payload.splitType, none, Sets how to split incoming messages. Supported split types are 'regex' and 'json_array'. Use 'none' for no splitting. +payload.splitType.regex.pattern, \n (newline), A regex based on which incoming requests will be split into multiple outgoing messages prometheus.port, 1234, Port used by the server that provides DropWizard metrics |=== diff --git a/etc/config.properties b/etc/config.properties index 1e5ea813..28090035 100644 --- a/etc/config.properties +++ b/etc/config.properties @@ -20,7 +20,7 @@ credentials.file=etc/credentials.json lookups.hostname.file=etc/hostname.json lookups.appname.file=etc/appname.json -payload.splitRegex=\n -payload.splitEnabled=false +payload.splitType=none +payload.splitType.regex.pattern=\n prometheus.port=1234 diff --git a/pom.xml b/pom.xml index 36bf106a..bb715064 100644 --- a/pom.xml +++ b/pom.xml @@ -140,6 +140,17 @@ log4j-core ${log4j.version} + + + jakarta.json + jakarta.json-api + 2.1.3 + + + org.eclipse.parsson + parsson + 1.1.7 + org.junit.jupiter @@ -159,6 +170,13 @@ rlp_03 ${rlp_03.version} + + + com.teragrep + rlo_06 + 9.0.1 + test + org.elasticsearch @@ -231,6 +249,7 @@ src/main/assembly/jar-with-dependencies.xml src/main/resources/* src/test/resources/certificates/* + src/test/resources/properties/* README.adoc diff --git a/src/main/java/com/teragrep/lsh_01/HttpInitializer.java b/src/main/java/com/teragrep/lsh_01/HttpInitializer.java index 07e3e76f..1bc29734 100644 --- a/src/main/java/com/teragrep/lsh_01/HttpInitializer.java +++ b/src/main/java/com/teragrep/lsh_01/HttpInitializer.java @@ -20,6 +20,7 @@ package com.teragrep.lsh_01; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import com.teragrep.lsh_01.util.LoggingHttpObjectAggregator; import com.teragrep.lsh_01.util.SslHandlerProvider; import io.netty.channel.ChannelInitializer; diff --git a/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java b/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java index 3da8f50d..e54a8db7 100644 --- a/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java +++ b/src/main/java/com/teragrep/lsh_01/HttpServerHandler.java @@ -20,6 +20,7 @@ package com.teragrep.lsh_01; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; diff --git a/src/main/java/com/teragrep/lsh_01/Main.java b/src/main/java/com/teragrep/lsh_01/Main.java index 92496397..bcb99a92 100644 --- a/src/main/java/com/teragrep/lsh_01/Main.java +++ b/src/main/java/com/teragrep/lsh_01/Main.java @@ -27,24 +27,34 @@ import com.teragrep.lsh_01.metrics.JmxReport; import com.teragrep.lsh_01.metrics.Report; import com.teragrep.lsh_01.metrics.Slf4jReport; +import com.teragrep.lsh_01.conversion.*; import com.teragrep.lsh_01.pool.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.Map; public class Main { private final static Logger LOGGER = LogManager.getLogger(Main.class); public static void main(String[] args) { + Map propsMap; + try { + propsMap = new PathProperties(System.getProperty("properties.file", "etc/config.properties")) + .deepCopyAsUnmodifiableMap(); + } + catch (IOException e) { + throw new IllegalArgumentException("Can't find properties file: ", e); + } + NettyConfig nettyConfig = new NettyConfig(); RelpConfig relpConfig = new RelpConfig(); SecurityConfig securityConfig = new SecurityConfig(); BasicAuthentication basicAuthentication = new BasicAuthenticationFactory().create(); InternalEndpointUrlConfig internalEndpointUrlConfig = new InternalEndpointUrlConfig(); LookupConfig lookupConfig = new LookupConfig(); - PayloadConfig payloadConfig = new PayloadConfig(); MetricsConfig metricsConfig = new MetricsConfig(); try { nettyConfig.validate(); @@ -52,7 +62,6 @@ public static void main(String[] args) { securityConfig.validate(); internalEndpointUrlConfig.validate(); lookupConfig.validate(); - payloadConfig.validate(); metricsConfig.validate(); } catch (IllegalArgumentException e) { @@ -63,7 +72,6 @@ public static void main(String[] args) { LOGGER.info("Got relp config: <[{}]>", relpConfig); LOGGER.info("Got internal endpoint config: <[{}]>", internalEndpointUrlConfig); LOGGER.info("Got lookup table config: <[{}]>", lookupConfig); - LOGGER.info("Got payload config: <[{}]>", payloadConfig); LOGGER.info("Authentication required: <[{}]>", securityConfig.authRequired); // metrics @@ -76,14 +84,14 @@ public static void main(String[] args) { RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(relpConfig, metricRegistry); Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); - IMessageHandler relpConversion = new MetricRelpConversion( - new RelpConversion(pool, securityConfig, basicAuthentication, lookupConfig, payloadConfig), + IMessageHandler conversion = new MetricRelpConversion( + new ConversionFactory(propsMap, pool, securityConfig, basicAuthentication, lookupConfig).conversion(), metricRegistry ); try ( HttpServer server = new MetricHttpServer( - new NettyHttpServer(nettyConfig, relpConversion, null, 200, internalEndpointUrlConfig), + new NettyHttpServer(nettyConfig, conversion, null, 200, internalEndpointUrlConfig), report ) ) { diff --git a/src/main/java/com/teragrep/lsh_01/MessageProcessor.java b/src/main/java/com/teragrep/lsh_01/MessageProcessor.java index 8476dd13..dfd240ff 100644 --- a/src/main/java/com/teragrep/lsh_01/MessageProcessor.java +++ b/src/main/java/com/teragrep/lsh_01/MessageProcessor.java @@ -21,6 +21,7 @@ import com.teragrep.lsh_01.authentication.*; import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandlerContext; diff --git a/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java b/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java index 353bc955..07c2c0a1 100644 --- a/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java +++ b/src/main/java/com/teragrep/lsh_01/NettyHttpServer.java @@ -21,6 +21,7 @@ import com.teragrep.lsh_01.config.InternalEndpointUrlConfig; import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelOption; diff --git a/src/main/java/com/teragrep/lsh_01/Payload.java b/src/main/java/com/teragrep/lsh_01/Payload.java deleted file mode 100644 index 73b079b6..00000000 --- a/src/main/java/com/teragrep/lsh_01/Payload.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - logstash-http-input to syslog bridge - Copyright 2024 Suomen Kanuuna Oy - - Derivative Work of Elasticsearch - Copyright 2012-2015 Elasticsearch - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.teragrep.lsh_01; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -/** - * A message from a log source - */ -final public class Payload { - - private final String body; - private final Pattern splitPattern; - - public Payload(String body, Pattern splitPattern) { - this.body = body; - this.splitPattern = splitPattern; - } - - /** - * Splits the payload into multiple payloads if there is a defined split regex in the body. - * - * @return list of Payloads - */ - public List split() { - ArrayList payloads = new ArrayList<>(); - - String[] messages = splitPattern.split(body); - - for (String message : messages) { - payloads.add(new Payload(message, splitPattern)); - } - - return payloads; - } - - /** - * Takes the message from the payload. - * - * @return message body - */ - public String take() { - return body; - } -} diff --git a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java index 1daafac1..49d36ba8 100644 --- a/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java +++ b/src/main/java/com/teragrep/lsh_01/authentication/BasicAuthentication.java @@ -24,6 +24,7 @@ import org.apache.logging.log4j.Logger; import java.util.Base64; +import java.util.Objects; public class BasicAuthentication { @@ -67,4 +68,19 @@ public Subject asSubject(String token) { return subjectStub; } } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final BasicAuthentication cast = (BasicAuthentication) o; + return decoder.equals(cast.decoder) && credentialLookup.equals(cast.credentialLookup); + } + + @Override + public int hashCode() { + return Objects.hash(decoder, credentialLookup); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/Configuration.java b/src/main/java/com/teragrep/lsh_01/config/Configuration.java new file mode 100644 index 00000000..b66515f5 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/Configuration.java @@ -0,0 +1,34 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import java.io.IOException; +import java.util.Map; + +public interface Configuration { + + /** + * Get configuration as an unmodifiable map so that it can't be altered anymore. + * + * @return configuration as an unmodifiable map + * @throws IOException if configuration file is not found + */ + Map deepCopyAsUnmodifiableMap() throws IOException; +} diff --git a/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java new file mode 100644 index 00000000..8d32d551 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/ConversionFactory.java @@ -0,0 +1,145 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import com.teragrep.lsh_01.authentication.BasicAuthentication; +import com.teragrep.lsh_01.conversion.IMessageHandler; +import com.teragrep.lsh_01.conversion.JsonConversion; +import com.teragrep.lsh_01.conversion.RegexConversion; +import com.teragrep.lsh_01.conversion.RelpConversion; +import com.teragrep.lsh_01.pool.IManagedRelpConnection; +import com.teragrep.lsh_01.pool.Pool; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +public final class ConversionFactory { + + private final static Logger LOGGER = LogManager.getLogger(ConversionFactory.class); + + private final String splitType; + private final String regexPattern; + private final Pool pool; + private final SecurityConfig securityConfig; + private final BasicAuthentication basicAuthentication; + private final LookupConfig lookupConfig; + + public ConversionFactory( + Map configuration, + Pool pool, + SecurityConfig securityConfig, + BasicAuthentication basicAuthentication, + LookupConfig lookupConfig + ) { + // if system property is not specified, defaults to config file (the Map) + this( + configuration.get("payload.splitType"), + configuration.get("payload.splitType.regex.pattern"), + pool, + securityConfig, + basicAuthentication, + lookupConfig + ); + } + + public ConversionFactory( + String splitType, + String regexPattern, + Pool pool, + SecurityConfig securityConfig, + BasicAuthentication basicAuthentication, + LookupConfig lookupConfig + ) { + this.splitType = splitType; + this.regexPattern = regexPattern; + this.pool = pool; + this.securityConfig = securityConfig; + this.basicAuthentication = basicAuthentication; + this.lookupConfig = lookupConfig; + } + + public IMessageHandler conversion() { + LOGGER + .info( + "Creating IMessageHandler with given configuration: payload.splitType=<[{}]>, payload.splitType.regex.pattern=<[{}]>", + splitType, regexPattern + ); + + validateConfiguration(); + + IMessageHandler conversion = new RelpConversion(pool, securityConfig, basicAuthentication, lookupConfig); + + // apply splitting if configured. "none" value is skipped + switch (splitType) { + case "regex": + conversion = new RegexConversion(conversion, regexPattern); + break; + case "json_array": + conversion = new JsonConversion(conversion); + break; + } + + return conversion; + } + + private void validateConfiguration() { + switch (splitType) { + case "regex": + try { + Pattern.compile(regexPattern); + } + catch (PatternSyntaxException e) { + throw new IllegalArgumentException( + "Configuration has an invalid regex (payload.splitType.regex.pattern): " + regexPattern + ); + } + break; + case "json_array": + case "none": + break; + default: + throw new IllegalArgumentException( + "Configuration has an invalid splitType: " + splitType + + ". Has to be 'regex', 'json_array' or 'none'." + ); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final ConversionFactory cast = (ConversionFactory) o; + return splitType.equals(cast.splitType) && regexPattern.equals(cast.regexPattern) && pool + .equals(cast.pool) && securityConfig.equals(cast.securityConfig) + && basicAuthentication.equals(cast.basicAuthentication) && lookupConfig.equals(cast.lookupConfig); + } + + @Override + public int hashCode() { + return Objects.hash(splitType, regexPattern, pool, securityConfig, basicAuthentication, lookupConfig); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java index b3059fa3..fabe9a6a 100644 --- a/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/LookupConfig.java @@ -19,6 +19,8 @@ */ package com.teragrep.lsh_01.config; +import java.util.Objects; + public class LookupConfig implements Validateable { public final String hostnamePath; @@ -40,4 +42,19 @@ public void validate() { public String toString() { return "LookupConfig{" + "hostnamePath='" + hostnamePath + '\'' + ", appNamePath='" + appNamePath + '\'' + '}'; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final LookupConfig cast = (LookupConfig) o; + return hostnamePath.equals(cast.hostnamePath) && appNamePath.equals(cast.appNamePath); + } + + @Override + public int hashCode() { + return Objects.hash(hostnamePath, appNamePath); + } } diff --git a/src/main/java/com/teragrep/lsh_01/config/PathProperties.java b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java new file mode 100644 index 00000000..e91a6ebe --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/config/PathProperties.java @@ -0,0 +1,73 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.config; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; +import java.util.stream.Collectors; + +public final class PathProperties implements Configuration { + + private final File file; + + public PathProperties(String fileName) { + this.file = new File(fileName); + } + + public PathProperties(File file) { + this.file = file; + } + + @Override + public Map deepCopyAsUnmodifiableMap() throws IOException { + Properties properties = new Properties(); + try (final InputStream in = Files.newInputStream(file.toPath())) { + properties.load(in); + } + return Collections + .unmodifiableMap( + properties + .entrySet() + .stream() + .collect(Collectors.toMap(k -> k.getKey().toString(), k -> k.getValue().toString())) + ); + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + PathProperties that = (PathProperties) o; + return file.equals(that.file); + } + + @Override + public int hashCode() { + return Objects.hashCode(file); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java b/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java deleted file mode 100644 index dc2260ab..00000000 --- a/src/main/java/com/teragrep/lsh_01/config/PayloadConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - logstash-http-input to syslog bridge - Copyright 2024 Suomen Kanuuna Oy - - Derivative Work of Elasticsearch - Copyright 2012-2015 Elasticsearch - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.teragrep.lsh_01.config; - -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -public class PayloadConfig implements Validateable { - - public final String splitRegex; - public final boolean splitEnabled; - - public PayloadConfig() { - PropertiesReaderUtilityClass propertiesReader = new PropertiesReaderUtilityClass( - System.getProperty("properties.file", "etc/config.properties") - ); - splitRegex = propertiesReader.getStringProperty("payload.splitRegex"); - splitEnabled = propertiesReader.getBooleanProperty("payload.splitEnabled"); - } - - @Override - public void validate() { - if (splitEnabled) { - try { - Pattern.compile(splitRegex); - } - catch (PatternSyntaxException e) { - throw new IllegalArgumentException( - "Configuration has an invalid regex (payload.splitRegex): " + splitRegex - ); - } - } - } - - @Override - public String toString() { - return "PayloadConfig{" + "splitRegex=" + splitRegex + ", splitEnabled=" + splitEnabled + '}'; - } -} diff --git a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java index cfca2cbb..fd020f3a 100644 --- a/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java +++ b/src/main/java/com/teragrep/lsh_01/config/SecurityConfig.java @@ -19,6 +19,8 @@ */ package com.teragrep.lsh_01.config; +import java.util.Objects; + public class SecurityConfig implements Validateable { public final boolean authRequired; @@ -34,4 +36,19 @@ public SecurityConfig() { public void validate() { } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final SecurityConfig cast = (SecurityConfig) o; + return authRequired == cast.authRequired; + } + + @Override + public int hashCode() { + return Objects.hashCode(authRequired); + } } diff --git a/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java new file mode 100644 index 00000000..b95e9441 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/DefaultPayload.java @@ -0,0 +1,55 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public final class DefaultPayload implements Payload { + + private final String message; + + public DefaultPayload(String message) { + this.message = message; + } + + @Override + public List messages() { + return Collections.singletonList(message); + } + + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final DefaultPayload cast = (DefaultPayload) object; + return message.equals(cast.message); + } + + @Override + public int hashCode() { + return Objects.hashCode(message); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/IMessageHandler.java b/src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java similarity index 97% rename from src/main/java/com/teragrep/lsh_01/IMessageHandler.java rename to src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java index 8b4b4cf8..cc51ba50 100644 --- a/src/main/java/com/teragrep/lsh_01/IMessageHandler.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/IMessageHandler.java @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.teragrep.lsh_01.authentication.Subject; diff --git a/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java new file mode 100644 index 00000000..0df2ca2a --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/JsonConversion.java @@ -0,0 +1,70 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import com.teragrep.lsh_01.authentication.Subject; + +import java.util.Map; + +/** + * Decorator for IMessageHandler that splits messages arriving as an array of JSON objects. + */ +public final class JsonConversion implements IMessageHandler { + + private final IMessageHandler conversion; + + public JsonConversion(IMessageHandler conversion) { + this.conversion = conversion; + } + + @Override + public boolean onNewMessage(Subject subject, Map headers, String body) { + JsonPayload originalPayload = new JsonPayload(new DefaultPayload(body)); + + boolean msgSent = true; + for (String message : originalPayload.messages()) { // each object individually as a String + if (!conversion.onNewMessage(subject, headers, message)) { + msgSent = false; + } + } + + return msgSent; + } + + @Override + public Subject asSubject(String token) { + return conversion.asSubject(token); + } + + @Override + public boolean requiresToken() { + return conversion.requiresToken(); + } + + @Override + public IMessageHandler copy() { + return new JsonConversion(conversion.copy()); + } + + @Override + public Map responseHeaders() { + return conversion.responseHeaders(); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java new file mode 100644 index 00000000..64be5bd0 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/JsonPayload.java @@ -0,0 +1,76 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import jakarta.json.*; + +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * A Json array payload splittable into individual json objects. + */ +public final class JsonPayload implements Payload { + + private final Payload payload; + + public JsonPayload(Payload payload) { + this.payload = payload; + } + + /** + * Splits the array of JSON objects into payloads with one object each. Has a side effect of removing whitespace + * from the payloads because of jsonObject.toString(). + * + * @return list of messages + */ + @Override + public List messages() { + List allMessages = new ArrayList<>(); + for (String message : payload.messages()) { + JsonReader reader = Json.createReader(new StringReader(message)); + JsonArray payloadMessages = reader.readArray(); + + // transform all json objects into DefaultPayloads and return the list + allMessages.addAll(payloadMessages.getValuesAs(JsonObject::toString)); + } + + return allMessages; + } + + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final JsonPayload cast = (JsonPayload) object; + return payload.equals(cast.payload); + } + + @Override + public int hashCode() { + return Objects.hashCode(payload); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java similarity index 98% rename from src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java rename to src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java index 9d58257e..7843577b 100644 --- a/src/main/java/com/teragrep/lsh_01/MetricRelpConversion.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/MetricRelpConversion.java @@ -17,7 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SlidingWindowReservoir; diff --git a/src/main/java/com/teragrep/lsh_01/conversion/Payload.java b/src/main/java/com/teragrep/lsh_01/conversion/Payload.java new file mode 100644 index 00000000..554f244c --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/Payload.java @@ -0,0 +1,35 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import java.util.List; + +/** + * A message from a log source + */ +public interface Payload { + + /** + * Takes the message(s) from the payload. + * + * @return list of messages + */ + List messages(); +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java new file mode 100644 index 00000000..9c32ffb0 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/RegexConversion.java @@ -0,0 +1,77 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import com.teragrep.lsh_01.authentication.Subject; + +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Decorator for IMessageHandler that splits messages based on given regex. + */ +public final class RegexConversion implements IMessageHandler { + + private final IMessageHandler conversion; + private final Pattern pattern; + + public RegexConversion(IMessageHandler conversion, String regex) { + this(conversion, Pattern.compile(regex)); + } + + public RegexConversion(IMessageHandler conversion, Pattern pattern) { + this.conversion = conversion; + this.pattern = pattern; + } + + @Override + public boolean onNewMessage(Subject subject, Map headers, String body) { + RegexPayload originalPayload = new RegexPayload(new DefaultPayload(body), pattern); + + boolean msgSent = true; + for (String message : originalPayload.messages()) { // process each split message individually + if (!conversion.onNewMessage(subject, headers, message)) { + msgSent = false; + } + } + + return msgSent; + } + + @Override + public Subject asSubject(String token) { + return conversion.asSubject(token); + } + + @Override + public boolean requiresToken() { + return conversion.requiresToken(); + } + + @Override + public IMessageHandler copy() { + return new RegexConversion(conversion.copy(), pattern); + } + + @Override + public Map responseHeaders() { + return conversion.responseHeaders(); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java new file mode 100644 index 00000000..84a3a7b6 --- /dev/null +++ b/src/main/java/com/teragrep/lsh_01/conversion/RegexPayload.java @@ -0,0 +1,73 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01.conversion; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; + +/** + * Payload splittable with a regex pattern. + */ +final public class RegexPayload implements Payload { + + private final Payload payload; + private final Pattern splitPattern; + + public RegexPayload(Payload payload, Pattern splitPattern) { + this.payload = payload; + this.splitPattern = splitPattern; + } + + /** + * Splits the payload into multiple payloads if there is a defined split regex in the body. + * + * @return list of Payloads + */ + @Override + public List messages() { + ArrayList allMessages = new ArrayList<>(); + + for (String message : payload.messages()) { + String[] payloadMessages = splitPattern.split(message); + allMessages.addAll(List.of(payloadMessages)); + } + + return allMessages; + } + + @Override + public boolean equals(final Object object) { + if (this == object) + return true; + if (object == null) + return false; + if (object.getClass() != this.getClass()) + return false; + final RegexPayload cast = (RegexPayload) object; + return payload.equals(cast.payload) && splitPattern.equals(cast.splitPattern); + } + + @Override + public int hashCode() { + return Objects.hash(payload, splitPattern); + } +} diff --git a/src/main/java/com/teragrep/lsh_01/RelpConversion.java b/src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java similarity index 80% rename from src/main/java/com/teragrep/lsh_01/RelpConversion.java rename to src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java index e1bccf2d..1303a65c 100644 --- a/src/main/java/com/teragrep/lsh_01/RelpConversion.java +++ b/src/main/java/com/teragrep/lsh_01/conversion/RelpConversion.java @@ -17,13 +17,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -package com.teragrep.lsh_01; +package com.teragrep.lsh_01.conversion; import com.teragrep.jlt_01.StringLookupTable; import com.teragrep.lsh_01.authentication.BasicAuthentication; import com.teragrep.lsh_01.authentication.Subject; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.lookup.LookupTableFactory; import com.teragrep.lsh_01.pool.*; @@ -35,7 +34,6 @@ import java.time.Instant; import java.util.HashMap; import java.util.Map; -import java.util.regex.Pattern; public class RelpConversion implements IMessageHandler { @@ -44,7 +42,6 @@ public class RelpConversion implements IMessageHandler { private final SecurityConfig securityConfig; private final BasicAuthentication basicAuthentication; private final LookupConfig lookupConfig; - private final PayloadConfig payloadConfig; private final StringLookupTable hostnameLookup; private final StringLookupTable appnameLookup; @@ -52,8 +49,7 @@ public RelpConversion( Pool relpConnectionPool, SecurityConfig securityConfig, BasicAuthentication basicAuthentication, - LookupConfig lookupConfig, - PayloadConfig payloadConfig + LookupConfig lookupConfig ) { this.relpConnectionPool = relpConnectionPool; this.securityConfig = securityConfig; @@ -61,26 +57,13 @@ public RelpConversion( this.lookupConfig = lookupConfig; this.hostnameLookup = new LookupTableFactory().create(lookupConfig.hostnamePath); this.appnameLookup = new LookupTableFactory().create(lookupConfig.appNamePath); - this.payloadConfig = payloadConfig; } public boolean onNewMessage(Subject subject, Map headers, String body) { try { - if (payloadConfig.splitEnabled) { - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload originalPayload = new Payload(body, splitPattern); - - for (Payload payload : originalPayload.split()) { - sendMessage( - payload.take(), headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) - ); - } - } - else { - sendMessage( - body, headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) - ); - } + sendMessage( + body, headers, subject.subject(), hostnameLookup.lookup(subject.subject()), appnameLookup.lookup(subject.subject()) + ); } catch (Exception e) { LOGGER.error("Unexpected error when sending a message: <{}>", e.getMessage(), e); @@ -99,7 +82,7 @@ public boolean requiresToken() { public RelpConversion copy() { LOGGER.debug("RelpConversion.copy called"); - return new RelpConversion(relpConnectionPool, securityConfig, basicAuthentication, lookupConfig, payloadConfig); + return new RelpConversion(relpConnectionPool, securityConfig, basicAuthentication, lookupConfig); } public Map responseHeaders() { diff --git a/src/main/java/com/teragrep/lsh_01/pool/Pool.java b/src/main/java/com/teragrep/lsh_01/pool/Pool.java index e3808e45..8bbb8a2e 100644 --- a/src/main/java/com/teragrep/lsh_01/pool/Pool.java +++ b/src/main/java/com/teragrep/lsh_01/pool/Pool.java @@ -108,5 +108,4 @@ public void close() { // close all that are in the pool right now offer(stub); } - } diff --git a/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java b/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java index 86f79403..83ebb14e 100644 --- a/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java +++ b/src/main/java/com/teragrep/lsh_01/pool/RelpConnectionFactory.java @@ -30,6 +30,10 @@ public class RelpConnectionFactory implements Supplier { private final RelpConfig relpConfig; private final MetricRegistry metricRegistry; + public RelpConnectionFactory(RelpConfig relpConfig) { // for testing, new metric registry can be used + this(relpConfig, new MetricRegistry()); + } + public RelpConnectionFactory(RelpConfig relpConfig, MetricRegistry metricRegistry) { this.relpConfig = relpConfig; this.metricRegistry = metricRegistry; diff --git a/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java b/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java new file mode 100644 index 00000000..e361f534 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/BasicAuthenticationTest.java @@ -0,0 +1,77 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.jai_02.CredentialLookup; +import com.teragrep.lsh_01.authentication.BasicAuthentication; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.FileReader; + +public class BasicAuthenticationTest { + + @Test + public void testEquals() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials); + + Assertions.assertEquals(auth1, auth2); + } + + @Test + public void testNotEquals() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + String credentialsFile2 = "src/test/resources/exampleCredentials.json"; + BufferedReader br2 = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile2))); + CredentialLookup credentials2 = new CredentialLookup(br2); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials2); + + Assertions.assertNotEquals(auth1, auth2); + } + + @Test + public void testHashCode() { + String credentialsFile = "src/test/resources/credentials.json"; + BufferedReader br = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile))); + CredentialLookup credentials = new CredentialLookup(br); + + String credentialsFile2 = "src/test/resources/exampleCredentials.json"; + BufferedReader br2 = Assertions.assertDoesNotThrow(() -> new BufferedReader(new FileReader(credentialsFile2))); + CredentialLookup credentials2 = new CredentialLookup(br2); + + BasicAuthentication auth1 = new BasicAuthentication(credentials); + BasicAuthentication auth2 = new BasicAuthentication(credentials); + BasicAuthentication auth3 = new BasicAuthentication(credentials2); + + Assertions.assertEquals(auth1.hashCode(), auth2.hashCode()); + Assertions.assertNotEquals(auth1.hashCode(), auth3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java new file mode 100644 index 00000000..da282273 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/ConversionFactoryTest.java @@ -0,0 +1,252 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.authentication.BasicAuthentication; +import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; +import com.teragrep.lsh_01.config.*; +import com.teragrep.lsh_01.pool.IManagedRelpConnection; +import com.teragrep.lsh_01.pool.ManagedRelpConnectionStub; +import com.teragrep.lsh_01.pool.Pool; +import com.teragrep.lsh_01.pool.RelpConnectionFactory; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ConversionFactoryTest { + + @Test + public void testInvalidSplitRegex() { + String regexPattern = "(a*b{)"; + String splitType = "regex"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + IllegalArgumentException e = Assertions + .assertThrows(IllegalArgumentException.class, conversionFactory::conversion); + + Assertions + .assertEquals( + "Configuration has an invalid regex (payload.splitType.regex.pattern): " + regexPattern, + e.getMessage() + ); + } + + @Test + public void testValidSplitRegex() { + String regexPattern = "\\n"; + String splitType = "regex"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertDoesNotThrow(conversionFactory::conversion); + } + + @Test + public void testInvalidSplitType() { + String splitType = "invalid"; + String regexPattern = ""; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + IllegalArgumentException e = Assertions + .assertThrows(IllegalArgumentException.class, conversionFactory::conversion); + Assertions + .assertEquals( + "Configuration has an invalid splitType: " + splitType + + ". Has to be 'regex', 'json_array' or 'none'.", + e.getMessage() + ); + } + + @Test + public void testValidSplitType() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + + splitType = "regex"; + conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + + splitType = "none"; + conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + Assertions.assertDoesNotThrow(conversionFactory::conversion); + } + + @Test + public void testEqualConversionFactories() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + BasicAuthentication auth = new BasicAuthenticationFactory().create(); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + + ConversionFactory conversionFactoryCopy = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + + // calling functions should have no effect on an immutable object + conversionFactory.conversion(); + + Assertions.assertEquals(conversionFactory, conversionFactoryCopy); + } + + @Test + public void testNotEqualConversionFactories() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + + ConversionFactory conversionFactory = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + regexPattern = "\n"; + splitType = "regex"; + ConversionFactory conversionFactoryCopy = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertNotEquals(conversionFactory, conversionFactoryCopy); + } + + @Test + public void testHashCode() { + String regexPattern = ""; + String splitType = "json_array"; + + RelpConnectionFactory relpConnectionFactory = new RelpConnectionFactory(new RelpConfig()); + Pool pool = new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()); + BasicAuthentication auth = new BasicAuthenticationFactory().create(); + + ConversionFactory conversionFactory1 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + ConversionFactory conversionFactory2 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + auth, + new LookupConfig() + ); + + regexPattern = "\n"; + splitType = "regex"; + ConversionFactory conversionFactory3 = new ConversionFactory( + splitType, + regexPattern, + pool, + new SecurityConfig(), + new BasicAuthenticationFactory().create(), + new LookupConfig() + ); + + Assertions.assertEquals(conversionFactory1.hashCode(), conversionFactory2.hashCode()); + Assertions.assertNotEquals(conversionFactory1.hashCode(), conversionFactory3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/CredentialsTest.java b/src/test/java/com/teragrep/lsh_01/CredentialsTest.java index e1e95729..48bc3006 100644 --- a/src/test/java/com/teragrep/lsh_01/CredentialsTest.java +++ b/src/test/java/com/teragrep/lsh_01/CredentialsTest.java @@ -21,9 +21,9 @@ import com.codahale.metrics.MetricRegistry; import com.teragrep.lsh_01.authentication.BasicAuthentication; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.pool.*; @@ -62,8 +62,7 @@ public void testNoAuthRequired() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertFalse(relpConversion.requiresToken()); } @@ -90,8 +89,7 @@ public void testAuthRequired() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // FirstUser:VeryFirstPassword @@ -113,8 +111,7 @@ public void testValidBase64ButNoColon() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // Test @@ -136,8 +133,7 @@ public void testMultipleColons() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // UserWithColons:My:Password:Yay @@ -157,8 +153,7 @@ public void testInvalidBase64Auth() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions @@ -179,8 +174,7 @@ public void testNonBasicAuth() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions @@ -204,8 +198,7 @@ public void testWrongCredentials() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // SecondUser:WrongPassword -> Right user @@ -229,8 +222,7 @@ public void testEmptyUsername() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // :VeryFirstPassword -> Valid password, null username @@ -254,8 +246,7 @@ public void testEmptyPassword() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); // FirstUser: -> Valid username, null password @@ -277,8 +268,7 @@ public void testNullToken() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); Assertions.assertTrue(relpConversion.requiresToken()); IllegalArgumentException e = Assertions diff --git a/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java b/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java new file mode 100644 index 00000000..e8c18754 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/DefaultPayloadTest.java @@ -0,0 +1,56 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DefaultPayloadTest { + + @Test + public void testEquals() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload("payload"); + + // calling functions shouldn't have effect on an immutable object + payload1.messages(); + + Assertions.assertEquals(payload1, payload2); + } + + @Test + public void testNotEquals() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload(""); + + Assertions.assertNotEquals(payload1, payload2); + } + + @Test + public void testHashCode() { + DefaultPayload payload1 = new DefaultPayload("payload"); + DefaultPayload payload2 = new DefaultPayload("payload"); + DefaultPayload payload3 = new DefaultPayload(""); + + Assertions.assertEquals(payload1.hashCode(), payload2.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/EndToEndTest.java b/src/test/java/com/teragrep/lsh_01/EndToEndTest.java index 29f7f7f7..85e841fb 100644 --- a/src/test/java/com/teragrep/lsh_01/EndToEndTest.java +++ b/src/test/java/com/teragrep/lsh_01/EndToEndTest.java @@ -33,96 +33,44 @@ import java.util.List; import java.util.Random; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class EndToEndTest { private RelpServer relpServer; private NettyConfig nettyConfig; + private Thread program; @BeforeAll void setUp() throws InterruptedException { - System.setProperty("payload.splitEnabled", "true"); + System.setProperty("payload.splitType", "none"); System.setProperty("security.authRequired", "false"); System.setProperty("relp.port", "1601"); // Start listening to HTTP-requests - Thread program = new Thread(() -> Main.main(new String[] {})); + program = new Thread(() -> Main.main(new String[] {})); program.start(); Thread.sleep(3000); // wait for netty to start up - this.relpServer = new RelpServer(); - this.relpServer.setUpDefault(); + relpServer = new RelpServer(); + relpServer.setUpDefault(); - this.nettyConfig = new NettyConfig(); + nettyConfig = new NettyConfig(); } @AfterEach void reset() { - this.relpServer.clear(); + relpServer.clear(); } @AfterAll void tearDown() { - System.clearProperty("payload.splitEnabled"); + System.clearProperty("payload.splitType"); System.clearProperty("security.authRequired"); System.clearProperty("relp.port"); - this.relpServer.tearDown(); - } - - @Test - public void testSplittingMessage1() throws InterruptedException, ExecutionException { - String requestBody = "foofoo\nbar"; - - HttpClient httpClient = HttpClient.newHttpClient(); - - HttpRequest request = HttpRequest - .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) - .POST(HttpRequest.BodyPublishers.ofString(requestBody)) - .build(); - - CompletableFuture> response = httpClient - .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - - Assertions.assertEquals(200, response.get().statusCode()); - - List payloads = this.relpServer.payloads(); - - // assert that payload was correctly split into two - Assertions.assertEquals(2, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); - } - - @Test - public void testSplittingMessage2() throws InterruptedException, ExecutionException { - String requestBody = "foofoo\nbar\nfoo bar"; - - HttpClient httpClient = HttpClient.newHttpClient(); - - HttpRequest request = HttpRequest - .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) - .POST(HttpRequest.BodyPublishers.ofString(requestBody)) - .build(); - - CompletableFuture> response = httpClient - .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - - Assertions.assertEquals(200, response.get().statusCode()); - - List payloads = this.relpServer.payloads(); - - // assert that payload was correctly split into three parts - Assertions.assertEquals(3, payloads.size()); - Assertions.assertTrue(payloads.get(0).contains("foofoo")); - Assertions.assertFalse(payloads.get(0).contains("bar")); - Assertions.assertTrue(payloads.get(1).contains("bar")); - Assertions.assertFalse(payloads.get(1).contains("foofoo")); - Assertions.assertTrue(payloads.get(2).contains("foo bar")); + relpServer.tearDown(); + program.interrupt(); } @Test @@ -152,7 +100,7 @@ public void testNullHeaders() { } @Test - public void testMultipleRequests() throws ExecutionException, InterruptedException { + public void testMultipleRequests() { ArrayList requestBodies = new ArrayList<>(); HttpClient httpClient = HttpClient.newHttpClient(); @@ -169,7 +117,8 @@ public void testMultipleRequests() throws ExecutionException, InterruptedExcepti CompletableFuture> response = httpClient .sendAsync(request, HttpResponse.BodyHandlers.ofString()); - Assertions.assertEquals(200, response.get().statusCode()); + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); } List payloads = this.relpServer.payloads(); diff --git a/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java new file mode 100644 index 00000000..17a8d471 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/JsonPayloadTest.java @@ -0,0 +1,94 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import com.teragrep.lsh_01.conversion.JsonPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class JsonPayloadTest { + + @Test + public void testSingleSplitting() { + String message = "{\"foo\": 1}"; + String requestBody = "[\n" + message + "\n]"; + + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + List messages = payload.messages(); + + String expected = message.replaceAll("\\s", ""); + + Assertions.assertEquals(expected, messages.get(0)); + } + + @Test + public void testMultipleSplitting() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n]"; + + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + List messages = payload.messages(); + + String expected1 = message1.replaceAll("\\s", ""); + String expected2 = message2.replaceAll("\\s", ""); + + Assertions.assertEquals(2, messages.size()); + Assertions.assertEquals(expected1, messages.get(0)); + Assertions.assertEquals(expected2, messages.get(1)); + } + + @Test + public void testEquals() { + String requestBody = "[\n{\"foo\": 1}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload samePayload = new JsonPayload(new DefaultPayload(requestBody)); + + // public methods of JsonPayload shouldn't affect an immutable object + payload.messages(); + + Assertions.assertEquals(payload, samePayload); + } + + @Test + public void testNotEquals() { + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); + + Assertions.assertNotEquals(payload, difPayload); + } + + @Test + public void testHashCode() { + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + JsonPayload payload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload samePayload = new JsonPayload(new DefaultPayload(requestBody)); + JsonPayload difPayload = new JsonPayload(new DefaultPayload(difRequestBody)); + + Assertions.assertEquals(payload.hashCode(), samePayload.hashCode()); + Assertions.assertNotEquals(payload.hashCode(), difPayload.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java new file mode 100644 index 00000000..081abaa2 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/JsonSplittingTest.java @@ -0,0 +1,220 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; +import org.junit.jupiter.api.*; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class JsonSplittingTest { + + private RelpServer relpServer; + private Thread program; + + @BeforeAll + void setUp() throws InterruptedException { + // have to set config here before running Main + System.setProperty("properties.file", "src/test/resources/properties/jsonSplittingTest.properties"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("properties.file"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testJsonSplittingOneMessage() { // no splitting needed + String message = "{\"foo\": 1}"; + String requestBody = "[\n" + message + "\n]"; + String expected = message.replaceAll("\\s", ""); + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that there is just one payload with the correct message + Assertions.assertEquals(1, payloads.size()); + + RFC5424Frame frame = new RFC5424Frame(); + frame.load(new ByteArrayInputStream(payloads.get(0).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + + Assertions.assertEquals(expected, frame.msg.toString()); + } + + @Test + public void testJsonSplittingTwoMessages() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(message1.replaceAll("\\s", "")); + expectedList.add(message2.replaceAll("\\s", "")); + + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split + Assertions.assertEquals(expectedList.size(), payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(expectedList.size(), loops); + } + + @Test + public void testJsonSplittingThreeMessages() { + String message1 = "{\"foo\": 1}"; + String message2 = "{\"bar\": 2}"; + String message3 = "{\"foobar\": 3}"; + String requestBody = "[\n" + message1 + ",\n" + message2 + "\n, \n" + message3 + "\n]"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(message1.replaceAll("\\s", "")); + expectedList.add(message2.replaceAll("\\s", "")); + expectedList.add(message3.replaceAll("\\s", "")); + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split + Assertions.assertEquals(expectedList.size(), payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(expectedList.size(), loops); + } + + @Test + public void testJsonSplittingNestedObjects() { + String payload = "{\"foo\": {\"bar\": 2}}"; + String expected = payload.replaceAll("\\s", ""); + String requestBody = "[\n" + payload + "\n]"; + + NettyConfig nettyConfig = new NettyConfig(); + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split + Assertions.assertEquals(1, payloads.size()); + + RFC5424Frame frame = new RFC5424Frame(); + frame.load(new ByteArrayInputStream(payloads.get(0).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + + Assertions.assertEquals(expected, frame.msg.toString()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/LookupTest.java b/src/test/java/com/teragrep/lsh_01/LookupTest.java index 8c068966..26396980 100644 --- a/src/test/java/com/teragrep/lsh_01/LookupTest.java +++ b/src/test/java/com/teragrep/lsh_01/LookupTest.java @@ -21,11 +21,11 @@ import com.codahale.metrics.MetricRegistry; import com.teragrep.jlt_01.StringLookupTable; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.authentication.BasicAuthentication; import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.authentication.Subject; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; import com.teragrep.lsh_01.lookup.LookupTableFactory; @@ -69,8 +69,7 @@ public void testAppnameLookup() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // FirstUser:VeryFirstPassword! @@ -94,8 +93,7 @@ public void testHostnameLookup() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // FirstUser:VeryFirstPassword! @@ -121,8 +119,7 @@ public void testMissingLookups() { pool, securityConfig, basicAuthentication, - new LookupConfig(), - new PayloadConfig() + new LookupConfig() ); // MissingHostname:MyHostnameIsMissing diff --git a/src/test/java/com/teragrep/lsh_01/MetricTest.java b/src/test/java/com/teragrep/lsh_01/MetricTest.java index 11a829fe..4959e2bc 100644 --- a/src/test/java/com/teragrep/lsh_01/MetricTest.java +++ b/src/test/java/com/teragrep/lsh_01/MetricTest.java @@ -25,9 +25,11 @@ import com.teragrep.lsh_01.authentication.BasicAuthenticationFactory; import com.teragrep.lsh_01.authentication.SubjectAnonymous; import com.teragrep.lsh_01.config.LookupConfig; -import com.teragrep.lsh_01.config.PayloadConfig; import com.teragrep.lsh_01.config.RelpConfig; import com.teragrep.lsh_01.config.SecurityConfig; +import com.teragrep.lsh_01.conversion.IMessageHandler; +import com.teragrep.lsh_01.conversion.MetricRelpConversion; +import com.teragrep.lsh_01.conversion.RelpConversion; import com.teragrep.lsh_01.fakes.RelpConnectionFactoryFake; import com.teragrep.lsh_01.fakes.RelpConnectionFake; import com.teragrep.lsh_01.fakes.ResendingRelpConnectionFake; @@ -205,7 +207,7 @@ public void testSendLatencyMetric() { // latency of the whole process, message i // the message processing starts from RelpConversion IMessageHandler relpConversion = new MetricRelpConversion( - new RelpConversion(new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()), new SecurityConfig(), new BasicAuthenticationFactory().create(), new LookupConfig(), new PayloadConfig()), registry + new RelpConversion(new Pool<>(relpConnectionFactory, new ManagedRelpConnectionStub()), new SecurityConfig(), new BasicAuthenticationFactory().create(), new LookupConfig()), registry ); for (int i = 0; i < messages; i++) { diff --git a/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java new file mode 100644 index 00000000..ee226c17 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/MultithreadingEndToEndTest.java @@ -0,0 +1,139 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; +import org.junit.jupiter.api.*; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class MultithreadingEndToEndTest { + + private RelpServer relpServer; + private Thread program; + + @BeforeAll + void setUp() throws InterruptedException { + // have to set config here before running Main + System.setProperty("properties.file", "src/test/resources/properties/multithreadingTest.properties"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("properties.file"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testMultithreadingTenClients() { + List requestBodies = Collections.synchronizedList(new ArrayList<>()); + + HttpClient httpClient = HttpClient.newHttpClient(); + NettyConfig nettyConfig = new NettyConfig(); + + final int clients = 10; + final int messagesPerClient = 100; + + ExecutorService executor = Executors.newFixedThreadPool(clients); + List> futures = new ArrayList<>(); + + // Send messages in parallel + for (int i = 0; i < clients; i++) { + Future future = executor.submit(() -> { + for (int j = 0; j < messagesPerClient; j++) { + String requestBody = randomString(); + requestBodies.add(requestBody); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)).POST(HttpRequest.BodyPublishers.ofString(requestBody)).build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + // Assert that there is a successful response + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + } + }); + futures.add(future); + } + + // wait until all threads are done + for (Future future : futures) + Assertions.assertDoesNotThrow(() -> future.get()); + + List payloads = this.relpServer.payloads(); // get the results + Assertions.assertEquals(clients * messagesPerClient, payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (String payload : payloads) { + frame.load(new ByteArrayInputStream(payload.getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertTrue(requestBodies.contains(frame.msg.toString())); // order of payloads can differ + loops++; + } + Assertions.assertEquals(clients * messagesPerClient, loops); + } + + private String randomString() { + int leftLimit = 97; // letter 'a' + int rightLimit = 122; // letter 'z' + int targetStringLength = 10; + Random random = new Random(); + + return random + .ints(leftLimit, rightLimit + 1) + .limit(targetStringLength) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java b/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java new file mode 100644 index 00000000..94e43ffb --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/PathPropertiesTest.java @@ -0,0 +1,57 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.PathProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class PathPropertiesTest { + + @Test + public void testEquals() { + String fileName = "src/test/resources/properties/defaultTest.properties"; + PathProperties props1 = new PathProperties(fileName); + PathProperties props2 = new PathProperties(fileName); + + // calling functions shouldn't have effect on an immutable object + Assertions.assertDoesNotThrow(props1::deepCopyAsUnmodifiableMap); + + Assertions.assertEquals(props1, props2); + } + + @Test + public void testNotEquals() { + PathProperties props1 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props2 = new PathProperties("src/test/resources/properties/customTest.properties"); + + Assertions.assertNotEquals(props1, props2); + } + + @Test + public void testHashCode() { + PathProperties props1 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props2 = new PathProperties("src/test/resources/properties/defaultTest.properties"); + PathProperties props3 = new PathProperties("src/test/resources/properties/customTest.properties"); + + Assertions.assertEquals(props1.hashCode(), props2.hashCode()); + Assertions.assertNotEquals(props1.hashCode(), props3.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/PayloadTest.java b/src/test/java/com/teragrep/lsh_01/PayloadTest.java deleted file mode 100644 index 369f76fe..00000000 --- a/src/test/java/com/teragrep/lsh_01/PayloadTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - logstash-http-input to syslog bridge - Copyright 2024 Suomen Kanuuna Oy - - Derivative Work of Elasticsearch - Copyright 2012-2015 Elasticsearch - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -package com.teragrep.lsh_01; - -import com.teragrep.lsh_01.config.PayloadConfig; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.regex.Pattern; - -public class PayloadTest { - - @BeforeEach - public void addProperties() { - System.setProperty("payload.splitRegex", "\\n"); - System.setProperty("payload.splitEnabled", "false"); - } - - @AfterEach - public void cleanProperties() { - System.clearProperty("payload.splitRegex"); - System.clearProperty("payload.splitEnabled"); - } - - @Test - public void testDefaultSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - - String body = "foo\nbar\nfoobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(3, payloads.size()); - Assertions.assertEquals("foo", payloads.get(0).take()); - Assertions.assertEquals("bar", payloads.get(1).take()); - Assertions.assertEquals("foobar", payloads.get(2).take()); - } - - @Test - public void testInvalidSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - System.setProperty("payload.splitRegex", "(a*b{)"); - - PayloadConfig payloadConfig = new PayloadConfig(); - Assertions.assertThrows(IllegalArgumentException.class, payloadConfig::validate); - } - - @Test - public void testValidSplitRegex() { - System.setProperty("payload.splitEnabled", "true"); - - PayloadConfig payloadConfig = new PayloadConfig(); - Assertions.assertDoesNotThrow(payloadConfig::validate); - } - - @Test - public void testCustomSplitRegex() { - System.setProperty("payload.splitRegex", ","); - System.setProperty("payload.splitEnabled", "true"); - - String body = "foo,bar,foobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(3, payloads.size()); - Assertions.assertEquals("foo", payloads.get(0).take()); - Assertions.assertEquals("bar", payloads.get(1).take()); - Assertions.assertEquals("foobar", payloads.get(2).take()); - } - - @Test - public void testNoSplittingRequired() { - System.setProperty("payload.splitEnabled", "true"); - - String body = "foobar"; - PayloadConfig payloadConfig = new PayloadConfig(); - Pattern splitPattern = Pattern.compile(payloadConfig.splitRegex); - Payload payload = new Payload(body, splitPattern); - List payloads = payload.split(); - - Assertions.assertEquals(1, payloads.size()); - Assertions.assertEquals("foobar", payloads.get(0).take()); - } -} diff --git a/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java new file mode 100644 index 00000000..d6763b4b --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/RegexPayloadTest.java @@ -0,0 +1,111 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.conversion.DefaultPayload; +import com.teragrep.lsh_01.conversion.RegexPayload; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.regex.Pattern; + +public class RegexPayloadTest { + + @Test + public void testDefaultSplitRegex() { + String body = "foo\nbar\nfoobar"; + Pattern splitPattern = Pattern.compile("\\n"); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(3, messages.size()); + Assertions.assertEquals("foo", messages.get(0)); + Assertions.assertEquals("bar", messages.get(1)); + Assertions.assertEquals("foobar", messages.get(2)); + } + + @Test + public void testCustomSplitRegex() { + String body = "foo,bar,foobar"; + Pattern splitPattern = Pattern.compile(","); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(3, messages.size()); + Assertions.assertEquals("foo", messages.get(0)); + Assertions.assertEquals("bar", messages.get(1)); + Assertions.assertEquals("foobar", messages.get(2)); + } + + @Test + public void testNoSplittingRequired() { + String body = "foobar"; + Pattern splitPattern = Pattern.compile("\\n"); + RegexPayload payload = new RegexPayload(new DefaultPayload(body), splitPattern); + List messages = payload.messages(); + + Assertions.assertEquals(1, messages.size()); + Assertions.assertEquals("foobar", messages.get(0)); + } + + @Test + public void testObjectEquals() { + Pattern splitPattern = Pattern.compile("\\n"); + String requestBody = "[\n{\"foo\": 1}\n]"; + RegexPayload payload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload samePayload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + + // public methods of JsonPayload shouldn't affect an immutable object + payload.messages(); + + Assertions.assertEquals(payload, samePayload); + } + + @Test + public void testObjectNotEquals() { + Pattern splitPattern = Pattern.compile("\\n"); + Pattern difSplitPattern = Pattern.compile(","); + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + RegexPayload payload = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload difPayload = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); + RegexPayload difPattern = new RegexPayload(new DefaultPayload(requestBody), difSplitPattern); + + Assertions.assertNotEquals(payload, difPayload); + Assertions.assertNotEquals(payload, difPattern); + } + + @Test + public void testHashCode() { + Pattern splitPattern = Pattern.compile("\\n"); + Pattern difSplitPattern = Pattern.compile(","); + String requestBody = "[\n{\"foo\": 1}\n]"; + String difRequestBody = "[\n{\"bar\": 2}\n]"; + RegexPayload payload1 = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload payload2 = new RegexPayload(new DefaultPayload(requestBody), splitPattern); + RegexPayload payload3 = new RegexPayload(new DefaultPayload(difRequestBody), splitPattern); + RegexPayload payload4 = new RegexPayload(new DefaultPayload(requestBody), difSplitPattern); + + Assertions.assertEquals(payload1.hashCode(), payload2.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload3.hashCode()); + Assertions.assertNotEquals(payload1.hashCode(), payload4.hashCode()); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java new file mode 100644 index 00000000..8829d437 --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/RegexSplittingTest.java @@ -0,0 +1,153 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.NettyConfig; +import com.teragrep.lsh_01.util.RelpServer; +import com.teragrep.rlo_06.RFC5424Frame; +import org.junit.jupiter.api.*; + +import java.io.ByteArrayInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RegexSplittingTest { + + private RelpServer relpServer; + private Thread program; + private NettyConfig nettyConfig; + + @BeforeAll + void setUp() throws InterruptedException { + System.setProperty("properties.file", "src/test/resources/properties/regexSplittingTest.properties"); + + // Start listening to HTTP-requests + program = new Thread(() -> Main.main(new String[] {})); + program.start(); + + Thread.sleep(3000); // wait for netty to start up + + this.relpServer = new RelpServer(); + this.relpServer.setUpDefault(); + + this.nettyConfig = new NettyConfig(); + } + + @AfterEach + void reset() { + this.relpServer.clear(); + } + + @AfterAll + void tearDown() { + System.clearProperty("properties.file"); + this.relpServer.tearDown(); + program.interrupt(); + } + + @Test + public void testRegexSplittingTwoMessages() { + String expected1 = "foofoo"; + String expected2 = "bar"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(expected1); + expectedList.add(expected2); + + String requestBody = expected1 + "\n" + expected2; + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split into two + Assertions.assertEquals(expectedList.size(), payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(expectedList.size(), loops); + } + + @Test + public void testRegexSplittingThreeMessages() { + String expected1 = "foofoo"; + String expected2 = "bar"; + String expected3 = "foo bar"; + + ArrayList expectedList = new ArrayList<>(); + expectedList.add(expected1); + expectedList.add(expected2); + expectedList.add(expected3); + + String requestBody = expected1 + "\n" + expected2 + "\n" + expected3; + + HttpClient httpClient = HttpClient.newHttpClient(); + + HttpRequest request = HttpRequest + .newBuilder(URI.create("http://" + nettyConfig.listenAddress + ":" + nettyConfig.listenPort)) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + + CompletableFuture> response = httpClient + .sendAsync(request, HttpResponse.BodyHandlers.ofString()); + + int statusCode = Assertions.assertDoesNotThrow(() -> response.get().statusCode()); + Assertions.assertEquals(200, statusCode); + + List payloads = this.relpServer.payloads(); + + // assert that payload was correctly split into three parts + Assertions.assertEquals(expectedList.size(), payloads.size()); + + int loops = 0; + RFC5424Frame frame = new RFC5424Frame(); + for (int i = 0; i < payloads.size(); i++) { + frame.load(new ByteArrayInputStream(payloads.get(i).getBytes(StandardCharsets.UTF_8))); + Assertions.assertDoesNotThrow(frame::next); + Assertions.assertEquals(expectedList.get(i), frame.msg.toString()); + loops++; + } + Assertions.assertEquals(expectedList.size(), loops); + } +} diff --git a/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java b/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java new file mode 100644 index 00000000..eaf6357c --- /dev/null +++ b/src/test/java/com/teragrep/lsh_01/SecurityConfigTest.java @@ -0,0 +1,64 @@ +/* + logstash-http-input to syslog bridge + Copyright 2024 Suomen Kanuuna Oy + + Derivative Work of Elasticsearch + Copyright 2012-2015 Elasticsearch + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package com.teragrep.lsh_01; + +import com.teragrep.lsh_01.config.SecurityConfig; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class SecurityConfigTest { + + @Test + public void testEquals() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + SecurityConfig config2 = new SecurityConfig(); + + // calling functions shouldn't have effect on an immutable object + config1.validate(); + + Assertions.assertEquals(config1, config2); + System.clearProperty("properties.file"); + } + + @Test + public void testNotEquals() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + System.setProperty("properties.file", "src/test/resources/properties/customTest.properties"); + SecurityConfig config2 = new SecurityConfig(); + + Assertions.assertNotEquals(config1, config2); + System.clearProperty("properties.file"); + } + + @Test + public void testHashCode() { + System.setProperty("properties.file", "src/test/resources/properties/defaultTest.properties"); + SecurityConfig config1 = new SecurityConfig(); + SecurityConfig config2 = new SecurityConfig(); + System.setProperty("properties.file", "src/test/resources/properties/customTest.properties"); + SecurityConfig config3 = new SecurityConfig(); + + Assertions.assertEquals(config1.hashCode(), config2.hashCode()); + Assertions.assertNotEquals(config1.hashCode(), config3.hashCode()); + System.clearProperty("properties.file"); + } +} diff --git a/src/test/resources/exampleCredentials.json b/src/test/resources/exampleCredentials.json new file mode 100644 index 00000000..2862ea96 --- /dev/null +++ b/src/test/resources/exampleCredentials.json @@ -0,0 +1,6 @@ +[ + { + "identity": "ExampleUser", + "credential": "ExamplePassword" + } +] diff --git a/src/test/resources/properties/customTest.properties b/src/test/resources/properties/customTest.properties new file mode 100644 index 00000000..b2498a08 --- /dev/null +++ b/src/test/resources/properties/customTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=1234 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=1000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=regex +payload.splitType.regex.pattern=\n + +prometheus.port=1234 diff --git a/src/test/resources/properties/defaultTest.properties b/src/test/resources/properties/defaultTest.properties new file mode 100644 index 00000000..28090035 --- /dev/null +++ b/src/test/resources/properties/defaultTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=true + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=none +payload.splitType.regex.pattern=\n + +prometheus.port=1234 diff --git a/src/test/resources/properties/jsonSplittingTest.properties b/src/test/resources/properties/jsonSplittingTest.properties new file mode 100644 index 00000000..ae78facd --- /dev/null +++ b/src/test/resources/properties/jsonSplittingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=json_array +payload.splitType.regex.pattern=\n + +prometheus.port=1234 \ No newline at end of file diff --git a/src/test/resources/properties/multithreadingTest.properties b/src/test/resources/properties/multithreadingTest.properties new file mode 100644 index 00000000..977ea361 --- /dev/null +++ b/src/test/resources/properties/multithreadingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=10 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=none +payload.splitType.regex.pattern=\n + +prometheus.port=1234 diff --git a/src/test/resources/properties/regexSplittingTest.properties b/src/test/resources/properties/regexSplittingTest.properties new file mode 100644 index 00000000..9933e266 --- /dev/null +++ b/src/test/resources/properties/regexSplittingTest.properties @@ -0,0 +1,26 @@ +server.listenAddress=127.0.0.1 +server.listenPort=8080 +server.threads=1 +server.maxPendingRequests=128 +server.maxContentLength=262144 + +healthcheck.enabled=true +healthcheck.url=/healthcheck + +relp.target=127.0.0.1 +relp.port=1601 +relp.reconnectInterval=10000 +relp.rebindRequestAmount=1000000 +relp.rebindEnabled=false + +security.authRequired=false + +credentials.file=etc/credentials.json + +lookups.hostname.file=etc/hostname.json +lookups.appname.file=etc/appname.json + +payload.splitType=regex +payload.splitType.regex.pattern=\n + +prometheus.port=1234 \ No newline at end of file