ENH: Improve performance of TextAnalyzer.merge (both memory usage an…

…d speed)

ChangeLog.md

@@ -1,6 +1,10 @@
 
 ## Changes ##
 
+### 15.10.0
+ - ENH: Improve performance of TextAnalyzer.merge (both memory usage and speed)
+ - INT: Bump gradle to 8.11.1, org.springframework.boot to 3.4.0
+
 ### 15.9.0
  - ENH: Remove support for TextAnalyzer.Feature.LEGACY_JSON
  - ENH: cli option --json changed to --format json (default), added --format faker (to output a faker specification)

build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 tasks.register('examples') {

examples/datetraining/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/mergesimple/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/minicli/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/modebulk/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/moderecord/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/modestreaming/build.gradle

@@ -3,7 +3,7 @@ plugins {
 }
 
 wrapper {
-	gradleVersion = '8.11'
+	gradleVersion = '8.11.1'
 }
 
 repositories {

examples/web/build.gradle

@@ -1,5 +1,5 @@
 plugins {
-	id 'org.springframework.boot' version '3.3.5'
+	id 'org.springframework.boot' version '3.4.0'
 	id 'io.spring.dependency-management' version '1.1.6'
 	id "com.github.ben-manes.versions" version '0.51.0'
 	id 'java'
@@ -8,7 +8,7 @@ plugins {
 }
 
 wrapper {
-        gradleVersion = '8.11'
+        gradleVersion = '8.11.1'
 }
 
 group = 'com.cobber.fta'

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.11-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

settings.gradle

@@ -27,7 +27,7 @@ includeBuild 'examples/modestreaming'
 dependencyResolutionManagement {
 	versionCatalogs {
 		libs {
-			version('fta', '15.9.0')
+			version('fta', '15.10.0')
 			version('jacoco', '0.8.12')
 
 // https://mvnrepository.com/artifact/com.univocity/univocity-parsers

types/src/main/java/com/cobber/fta/CacheLRU.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2017-2024 Tim Segall
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cobber.fta;
+
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+public class CacheLRU<K, V> {
+    private final Cache<K, V> cache;
+
+    public CacheLRU(int capacity) {
+        this.cache = CacheBuilder.newBuilder()
+                .maximumSize(capacity)
+                .expireAfterAccess(10, TimeUnit.MINUTES)
+                .build();
+    }
+
+    public void put(K key, V value) {
+        cache.put(key, value);
+    }
+
+    public V get(K key) {
+        return cache.getIfPresent(key);
+    }
+
+    public void invalidate(K key) {
+        cache.invalidate(key);
+    }
+
+    public long size() {
+        return cache.size();
+    }
+}

types/src/main/java/com/cobber/fta/token/TokenStream.java

@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Set;
 
+import com.cobber.fta.CacheLRU;
 import com.cobber.fta.core.InternalErrorException;
 import com.cobber.fta.core.RegExpGenerator;
 import com.cobber.fta.core.Utils;
@@ -66,6 +67,8 @@ public class TokenStream {
 	/* The number of occurrences of this 'Pattern'. */
 	private long occurrences;
 
+	private final static CacheLRU<String, Automaton> cache = new CacheLRU<>(10);
+
 	/** The TokenStream that represents any input that is too long. */
 	public final static TokenStream ANYSHAPE = new TokenStream(Utils.repeat('x', Token.MAX_LENGTH + 1), 1);
 
@@ -462,7 +465,11 @@ public long getOccurrences() {
 	 * @return True if the TokenStream matches the supplied Regular Expression.
 	 */
 	public boolean matches(final String regExp) {
-		final Automaton automaton = new RegExp(RegExpGenerator.toAutomatonRE(regExp, false), RegExp.AUTOMATON).toAutomaton(new DatatypesAutomatonProvider());
+		Automaton automaton = cache.get(regExp);
+		if (automaton == null) {
+			automaton = new RegExp(RegExpGenerator.toAutomatonRE(regExp, false), RegExp.AUTOMATON).toAutomaton(new DatatypesAutomatonProvider());
+			cache.put(regExp, automaton);
+		}
 
 		return matches(automaton.getInitialState(), 0);
 	}

types/src/test/java/com/cobber/fta/AllocationTracker.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2017-2024 Tim Segall
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cobber.fta;
+
+import java.lang.management.ManagementFactory;
+
+import com.sun.management.ThreadMXBean;
+public class AllocationTracker {
+    ThreadMXBean threadMxBean = (ThreadMXBean) ManagementFactory.getThreadMXBean();
+
+    public AllocationTracker() {
+        threadMxBean.setThreadAllocatedMemoryEnabled(true);
+    }
+
+    public long getAllocated() {
+    	return threadMxBean.getCurrentThreadAllocatedBytes();
+    }
+}

types/src/test/java/com/cobber/fta/TestDistributions.java

@@ -767,73 +767,74 @@ public void leadingPlus() throws IOException, FTAException {
 	// FLAKY!!!!!
 	@Test(groups = { TestGroups.ALL, TestGroups.DISTRIBUTION })
 	public void normalCurve() throws IOException, FTAException {
-		for (int iter = 0; iter < 100; iter++) {
-		final TextAnalyzer analysis = new TextAnalyzer("normalCurve");
-		analysis.setLocale(Locale.forLanguageTag("en-US"));
-		final SecureRandom random = new SecureRandom();
-		final int SIZE = 100000;
-
-		for (int i = 0; i < SIZE; i++)
-			analysis.train(String.valueOf(random.nextGaussian()*100));
-
-		// Test pre getResult()
-		String serialized = analysis.serialize();
-		final TextAnalyzer hydrated = TextAnalyzer.deserialize(serialized);
-		assertTrue(analysis.equals(hydrated));
-
-		// Test a hydrated object
-		serialized = hydrated.serialize();
-		final TextAnalyzer rehydrated = TextAnalyzer.deserialize(serialized);
-		assertEquals(serialized, rehydrated.serialize());
-
-		final TextAnalysisResult result = rehydrated.getResult();
-
-		// Test post getResult()
-		serialized = rehydrated.serialize();
-		assertEquals(serialized, TextAnalyzer.deserialize(serialized).serialize());
-
-		assertEquals(result.getSampleCount(), SIZE);
-		assertEquals(result.getOutlierCount(), 0);
-		assertEquals(result.getMatchCount(), SIZE);
-		assertEquals(result.getNullCount(), 0);
-		assertEquals(result.getConfidence(), 1.0);
-		assertEquals(result.getType(), FTAType.DOUBLE);
-		assertEquals(result.getTypeModifier(), "SIGNED");
-
-		assertEquals(result.getMean(), 0.0, 1.0);
-		assertEquals(result.getStandardDeviation(), 100, 1);
-
-		final String q0_0 = result.getValueAtQuantile(0);
-		final String q0_5 = result.getValueAtQuantile(.5);
-		final String q1_0 = result.getValueAtQuantile(1.0);
-
-		// Median should be seriously close to 0
-		assertEquals(Double.parseDouble(q0_5), 0.0, 1.2);
-
-		// 3.5 Standard Deviations should cover low and high points
-		assertTrue(Double.parseDouble(q0_0) < -350);
-		assertTrue(Double.parseDouble(q1_0) > 350);
-
-		// 101 because we want 0.0 and 1.0 plus everything in between
-		final double[] percentiles = new double[101];
-		double value = 0.0;
-		for (int i = 0; i < 100; i++) {
-			percentiles[i] = value;
-			value += .01;
-		}
-		// Make sure the last one is precisely 1.0
-		percentiles[100] = 1.0;
-
-		final String[] answers = result.getValuesAtQuantiles(percentiles);
-		assertEquals(answers[0], q0_0);
-		assertEquals(answers[50], q0_5);
-		assertEquals(answers[100], q1_0);
-
-		for (int i = 10; i < 50; i++) {
-			double low = Double.parseDouble(answers[i]);
-			double high = Double.parseDouble(answers[100 - i]);
-//			System.err.printf("low: %f, high: %f\n", low, high);
-		}
+		final int ITERATIONS = 1;
+		for (int iter = 0; iter < ITERATIONS; iter++) {
+			final TextAnalyzer analysis = new TextAnalyzer("normalCurve");
+			analysis.setLocale(Locale.forLanguageTag("en-US"));
+			final SecureRandom random = new SecureRandom();
+			final int SIZE = 100000;
+
+			for (int i = 0; i < SIZE; i++)
+				analysis.train(String.valueOf(random.nextGaussian()*100));
+
+			// Test pre getResult()
+			String serialized = analysis.serialize();
+			final TextAnalyzer hydrated = TextAnalyzer.deserialize(serialized);
+			assertTrue(analysis.equals(hydrated));
+
+			// Test a hydrated object
+			serialized = hydrated.serialize();
+			final TextAnalyzer rehydrated = TextAnalyzer.deserialize(serialized);
+			assertEquals(serialized, rehydrated.serialize());
+
+			final TextAnalysisResult result = rehydrated.getResult();
+
+			// Test post getResult()
+			serialized = rehydrated.serialize();
+			assertEquals(serialized, TextAnalyzer.deserialize(serialized).serialize());
+
+			assertEquals(result.getSampleCount(), SIZE);
+			assertEquals(result.getOutlierCount(), 0);
+			assertEquals(result.getMatchCount(), SIZE);
+			assertEquals(result.getNullCount(), 0);
+			assertEquals(result.getConfidence(), 1.0);
+			assertEquals(result.getType(), FTAType.DOUBLE);
+			assertEquals(result.getTypeModifier(), "SIGNED");
+
+			assertEquals(result.getMean(), 0.0, 1.0);
+			assertEquals(result.getStandardDeviation(), 100, 1);
+
+			final String q0_0 = result.getValueAtQuantile(0);
+			final String q0_5 = result.getValueAtQuantile(.5);
+			final String q1_0 = result.getValueAtQuantile(1.0);
+
+			// Median should be seriously close to 0
+			assertEquals(Double.parseDouble(q0_5), 0.0, 1.2);
+
+			// 3.5 Standard Deviations should cover low and high points
+			assertTrue(Double.parseDouble(q0_0) < -350);
+			assertTrue(Double.parseDouble(q1_0) > 350);
+
+			// 101 because we want 0.0 and 1.0 plus everything in between
+			final double[] percentiles = new double[101];
+			double value = 0.0;
+			for (int i = 0; i < 100; i++) {
+				percentiles[i] = value;
+				value += .01;
+			}
+			// Make sure the last one is precisely 1.0
+			percentiles[100] = 1.0;
+
+			final String[] answers = result.getValuesAtQuantiles(percentiles);
+			assertEquals(answers[0], q0_0);
+			assertEquals(answers[50], q0_5);
+			assertEquals(answers[100], q1_0);
+
+			for (int i = 10; i < 50; i++) {
+				double low = Double.parseDouble(answers[i]);
+				double high = Double.parseDouble(answers[100 - i]);
+//				System.err.printf("low: %f, high: %f\n", low, high);
+			}
 		}
 	}