Fix teragrep exec syslog stream command not working with subsequent c…

…ommands when using host and port parameters (#67) * Unit tests for syslog stream command * Pop grammar mode after port number is given so that the parser recognizes following commands properly * Remove exception throwing from TeragrepSyntaxTests * Change TERAGREP_COMMAND_MODE_IP mode name to TERAGREP_COMMAND_IP_MODE and use the name as prefix in the tokens in that mode
teragrep · Aug 21, 2024 · 1983e46 · 1983e46
1 parent b02551e
commit 1983e46
Show file tree

Hide file tree

Showing 5 changed files with 176 additions and 39 deletions.
diff --git a/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4 b/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4
@@ -75,8 +75,8 @@ COMMAND_TERAGREP_MODE_KAFKA: 'KAFKA' | 'kafka';
 COMMAND_TERAGREP_MODE_EXPLAIN: 'explain';
 COMMAND_TERAGREP_MODE_OVERWRITE: 'overwrite=' -> pushMode(GET_BOOLEAN);
 COMMAND_TERAGREP_MODE_PARSER: 'parser';
-COMMAND_TERAGREP_MODE_HOST: 'host' -> pushMode(COMMAND_TERAGREP_MODE_IP);
-COMMAND_TERAGREP_MODE_PORT: 'port' -> pushMode(COMMAND_TERAGREP_MODE_IP);
+COMMAND_TERAGREP_MODE_HOST: 'host' -> pushMode(COMMAND_TERAGREP_IP_MODE);
+COMMAND_TERAGREP_MODE_PORT: 'port' -> pushMode(COMMAND_TERAGREP_IP_MODE);
 COMMAND_TERAGREP_MODE_DOT: '.';
 COMMAND_TERAGREP_MODE_TOKENIZER: 'tokenizer';
 COMMAND_TERAGREP_MODE_SYSLOG: 'syslog';
@@ -205,10 +205,10 @@ fragment CHAR
          |'\u007E' // ~
          |'\u007F'..'\uFFFF' // DEL .. inf
          ;
-mode COMMAND_TERAGREP_MODE_IP;
-COMMAND_TERAGREP_MODE_IPE_SPACE: SPACE -> channel(HIDDEN);
-COMMAND_TERAGREP_IP: DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT -> popMode;
+mode COMMAND_TERAGREP_IP_MODE;
+COMMAND_TERAGREP_IP_MODE_SPACE: SPACE -> channel(HIDDEN);
+COMMAND_TERAGREP_IP_MODE_IP: DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT -> popMode;
 fragment DIGIT: [0-9]+;
-COMMAND_TERAGREP_MODE_COMMA: '.';
-COMMAND_TERAGREP_MODE_PORT_NUM: DIGIT;
-COMMAND_TERAGREP_MODE_COMMENT: '<!--' .*? '-->' -> channel(DPLCOMMENT);
+COMMAND_TERAGREP_IP_MODE_COMMA: '.';
+COMMAND_TERAGREP_IP_MODE_PORT_NUM: DIGIT -> popMode;
+COMMAND_TERAGREP_IP_MODE_COMMENT: '<!--' .*? '-->' -> channel(DPLCOMMENT);
diff --git a/src/main/antlr4/imports/DPLParserTransform_teragrep.g4 b/src/main/antlr4/imports/DPLParserTransform_teragrep.g4
@@ -135,11 +135,11 @@ t_bloomOptionParameter
         ;
 
 t_hostParameter
-        : COMMAND_TERAGREP_MODE_HOST t_portParameter COMMAND_TERAGREP_MODE_PORT COMMAND_TERAGREP_MODE_PORT_NUM
+        : COMMAND_TERAGREP_MODE_HOST t_portParameter COMMAND_TERAGREP_MODE_PORT COMMAND_TERAGREP_IP_MODE_PORT_NUM
         ;
 
 t_portParameter
-        : COMMAND_TERAGREP_IP
+        : COMMAND_TERAGREP_IP_MODE_IP
         ;
 
 t_overwriteParameter

diff --git a/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java
@@ -47,6 +47,7 @@
 
 import com.teragrep.pth_03.ParserStructureTestingUtility;
 import com.teragrep.pth_03.ParserSyntaxTestingUtility;
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.w3c.dom.NodeList;
@@ -70,25 +71,27 @@ public class TeragrepSyntaxTests {
             "teragrep12",
             "teragrep_csv_schema",
             "teragrep_csv_header",
-            "teragrep_archive_summary"
+            "teragrep_archive_summary",
+            "teragrep_syslog_stream",
+            "teragrep_syslog_stream_host_port"
     })
-    public void teragrepSyntaxParseTest(String arg) throws Exception {
+    public void teragrepSyntaxParseTest(String arg) {
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         ParserSyntaxTestingUtility parserSyntaxTestingUtility
                 = new ParserSyntaxTestingUtility(fileName, false);
-        parserSyntaxTestingUtility.syntaxParseTest(arg);
+        Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg));
     }
 
     @ParameterizedTest
     @ValueSource(strings = {
             "teragrep",
     })
-    void xpathTest1(String arg) throws Exception {
+    void xpathTest1(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/value";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -97,12 +100,12 @@ void xpathTest1(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep",
     })
-    void xpathTest2(String arg) throws Exception {
+    void xpathTest2(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_getParameter/t_getTeragrepVersionParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -111,12 +114,12 @@ void xpathTest2(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep4",
     })
-    void xpathTest3(String arg) throws Exception {
+    void xpathTest3(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_kafkaSaveModeParameter/value[1]";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -125,12 +128,12 @@ void xpathTest3(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep_tokenizer",
     })
-    void xpathTestTokenizer(String arg) throws Exception {
+    void xpathTestTokenizer(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_tokenizerParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -139,12 +142,12 @@ void xpathTestTokenizer(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep7",
     })
-    void xpathTestDynatrace(String arg) throws Exception {
+    void xpathTestDynatrace(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_dynatraceParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -153,12 +156,12 @@ void xpathTestDynatrace(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep8",
     })
-    void xpathTestDynatraceWithOptionalParam(String arg) throws Exception {
+    void xpathTestDynatraceWithOptionalParam(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_dynatraceParameter/stringType";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -167,12 +170,12 @@ void xpathTestDynatraceWithOptionalParam(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep9",
     })
-    void xpathTestCsvSave(String arg) throws Exception {
+    void xpathTestCsvSave(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -181,12 +184,12 @@ void xpathTestCsvSave(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep10",
     })
-    void xpathTestJsonSave(String arg) throws Exception {
+    void xpathTestJsonSave(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -195,12 +198,12 @@ void xpathTestJsonSave(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep11",
     })
-    void xpathTestDefaultSave(String arg) throws Exception {
+    void xpathTestDefaultSave(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -209,25 +212,25 @@ void xpathTestDefaultSave(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep12",
     })
-    void xpathTestCsvLoad(String arg) throws Exception {
+    void xpathTestCsvLoad(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_hdfsFormatParameter";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
     @ParameterizedTest
     @ValueSource(strings = {
             "teragrep_csv_header",
     })
-    void xpathTestCsvLoadHeader(String arg) throws Exception {
+    void xpathTestCsvLoadHeader(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_headerParameter/booleanType";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -236,12 +239,12 @@ void xpathTestCsvLoadHeader(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep_csv_schema",
     })
-    void xpathTestCsvLoadSchema(String arg) throws Exception {
+    void xpathTestCsvLoadSchema(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_schemaParameter/stringType";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
@@ -250,13 +253,55 @@ void xpathTestCsvLoadSchema(String arg) throws Exception {
     @ValueSource(strings = {
             "teragrep_archive_summary",
     })
-    void xpathTestArchiveSummary(String arg) throws Exception {
+    void xpathTestArchiveSummary(String arg) {
         ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
         String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
         String xpathExp = "/root/transformStatement/teragrepTransformation/t_getParameter/t_getArchiveSummaryParameter/searchTransformationRoot/directoryStatement/directoryStatement/indexStatement/stringType";
 
-        NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false);
+        NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false));
         // Check that 1 found
         assertEquals(1,nodesA.getLength());
     }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "teragrep_syslog_stream",
+    })
+    void syslogStreamTest(String arg) { // includes an eval command in the end of the query to make sure commands given after syslog work too
+        ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
+        String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
+        String syslogPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter";
+        String evalPath = "/root/transformStatement/transformStatement/evalTransformation";
+
+        NodeList syslogNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, syslogPath, true));
+        NodeList evalNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, evalPath, false));
+
+        // Check that 1 found
+        assertEquals(1,syslogNodes.getLength());
+        assertEquals(1,evalNodes.getLength());
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = {
+            "teragrep_syslog_stream_host_port",
+    })
+    void syslogStreamWithHostAndPortTest(String arg) { // includes an eval command in the end of the query to make sure commands given after syslog work too
+        ParserStructureTestingUtility pstu = new ParserStructureTestingUtility();
+        String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt";
+        String syslogPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter";
+        String hostPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter/t_hostParameter";
+        String portPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter/t_hostParameter/t_portParameter";
+        String evalPath = "/root/transformStatement/transformStatement/evalTransformation";
+
+        NodeList syslogNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, syslogPath, false));
+        NodeList hostNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, hostPath, false));
+        NodeList portNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, portPath, false));
+        NodeList evalNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, evalPath, false));
+
+        // Check that 1 found
+        assertEquals(1,syslogNodes.getLength());
+        assertEquals(1,hostNodes.getLength());
+        assertEquals(1,portNodes.getLength());
+        assertEquals(1,evalNodes.getLength());
+    }
 }
diff --git a/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt b/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt
@@ -0,0 +1,46 @@
+<!-- /*
+ * Teragrep Data Processing Language Parser Library PTH-03
+ * Copyright (C) 2019, 2020, 2021, 2022  Suomen Kanuuna Oy
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://github.com/teragrep/teragrep/blob/main/LICENSE>.
+ *
+ *
+ * Additional permission under GNU Affero General Public License version 3
+ * section 7
+ *
+ * If you modify this Program, or any covered work, by linking or combining it
+ * with other code, such other code is not for that reason alone subject to any
+ * of the requirements of the GNU Affero GPL version 3 as long as this Program
+ * is the same Program as licensed from Suomen Kanuuna Oy without any additional
+ * modifications.
+ *
+ * Supplemented terms under GNU Affero General Public License version 3
+ * section 7
+ *
+ * Origin of the software must be attributed to Suomen Kanuuna Oy. Any modified
+ * versions must be marked as "Modified version of" The Program.
+ *
+ * Names of the licensors and authors may not be used for publicity purposes.
+ *
+ * No rights are granted for use of trade names, trademarks, or service marks
+ * which are in The Program if any.
+ *
+ * Licensee must indemnify licensors and authors for any liability that these
+ * contractual assumptions impose on licensors and authors.
+ *
+ * To the extent this program is licensed as part of the Commercial versions of
+ * Teragrep, the applicable Commercial License may apply to this file if you as
+ * a licensee so wish it.
+ */ -->
+| teragrep exec syslog stream | eval a=1