From 1983e46edb89e1831afc2759af91a10cd482d528 Mon Sep 17 00:00:00 2001 From: 51-code <146736881+51-code@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:36:33 +0300 Subject: [PATCH] Fix teragrep exec syslog stream command not working with subsequent commands when using host and port parameters (#67) * Unit tests for syslog stream command * Pop grammar mode after port number is given so that the parser recognizes following commands properly * Remove exception throwing from TeragrepSyntaxTests * Change TERAGREP_COMMAND_MODE_IP mode name to TERAGREP_COMMAND_IP_MODE and use the name as prefix in the tokens in that mode --- .../antlr4/imports/COMMAND_TERAGREP_MODE.g4 | 16 +-- .../imports/DPLParserTransform_teragrep.g4 | 4 +- .../pth_03/tests/TeragrepSyntaxTests.java | 103 +++++++++++++----- .../teragrep/teragrep_syslog_stream.txt | 46 ++++++++ .../teragrep_syslog_stream_host_port.txt | 46 ++++++++ 5 files changed, 176 insertions(+), 39 deletions(-) create mode 100644 src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt create mode 100644 src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream_host_port.txt diff --git a/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4 b/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4 index 3241f14..3bcfe0f 100644 --- a/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4 +++ b/src/main/antlr4/imports/COMMAND_TERAGREP_MODE.g4 @@ -75,8 +75,8 @@ COMMAND_TERAGREP_MODE_KAFKA: 'KAFKA' | 'kafka'; COMMAND_TERAGREP_MODE_EXPLAIN: 'explain'; COMMAND_TERAGREP_MODE_OVERWRITE: 'overwrite=' -> pushMode(GET_BOOLEAN); COMMAND_TERAGREP_MODE_PARSER: 'parser'; -COMMAND_TERAGREP_MODE_HOST: 'host' -> pushMode(COMMAND_TERAGREP_MODE_IP); -COMMAND_TERAGREP_MODE_PORT: 'port' -> pushMode(COMMAND_TERAGREP_MODE_IP); +COMMAND_TERAGREP_MODE_HOST: 'host' -> pushMode(COMMAND_TERAGREP_IP_MODE); +COMMAND_TERAGREP_MODE_PORT: 'port' -> pushMode(COMMAND_TERAGREP_IP_MODE); COMMAND_TERAGREP_MODE_DOT: '.'; COMMAND_TERAGREP_MODE_TOKENIZER: 'tokenizer'; COMMAND_TERAGREP_MODE_SYSLOG: 'syslog'; @@ -205,10 +205,10 @@ fragment CHAR |'\u007E' // ~ |'\u007F'..'\uFFFF' // DEL .. inf ; -mode COMMAND_TERAGREP_MODE_IP; -COMMAND_TERAGREP_MODE_IPE_SPACE: SPACE -> channel(HIDDEN); -COMMAND_TERAGREP_IP: DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_MODE_COMMA)DIGIT -> popMode; +mode COMMAND_TERAGREP_IP_MODE; +COMMAND_TERAGREP_IP_MODE_SPACE: SPACE -> channel(HIDDEN); +COMMAND_TERAGREP_IP_MODE_IP: DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT(COMMAND_TERAGREP_IP_MODE_COMMA)DIGIT -> popMode; fragment DIGIT: [0-9]+; -COMMAND_TERAGREP_MODE_COMMA: '.'; -COMMAND_TERAGREP_MODE_PORT_NUM: DIGIT; -COMMAND_TERAGREP_MODE_COMMENT: '' -> channel(DPLCOMMENT); +COMMAND_TERAGREP_IP_MODE_COMMA: '.'; +COMMAND_TERAGREP_IP_MODE_PORT_NUM: DIGIT -> popMode; +COMMAND_TERAGREP_IP_MODE_COMMENT: '' -> channel(DPLCOMMENT); diff --git a/src/main/antlr4/imports/DPLParserTransform_teragrep.g4 b/src/main/antlr4/imports/DPLParserTransform_teragrep.g4 index e437af7..1b14e92 100644 --- a/src/main/antlr4/imports/DPLParserTransform_teragrep.g4 +++ b/src/main/antlr4/imports/DPLParserTransform_teragrep.g4 @@ -135,11 +135,11 @@ t_bloomOptionParameter ; t_hostParameter - : COMMAND_TERAGREP_MODE_HOST t_portParameter COMMAND_TERAGREP_MODE_PORT COMMAND_TERAGREP_MODE_PORT_NUM + : COMMAND_TERAGREP_MODE_HOST t_portParameter COMMAND_TERAGREP_MODE_PORT COMMAND_TERAGREP_IP_MODE_PORT_NUM ; t_portParameter - : COMMAND_TERAGREP_IP + : COMMAND_TERAGREP_IP_MODE_IP ; t_overwriteParameter diff --git a/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java b/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java index 7e59e79..f537fed 100644 --- a/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java +++ b/src/test/java/com/teragrep/pth_03/tests/TeragrepSyntaxTests.java @@ -47,6 +47,7 @@ import com.teragrep.pth_03.ParserStructureTestingUtility; import com.teragrep.pth_03.ParserSyntaxTestingUtility; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.w3c.dom.NodeList; @@ -70,25 +71,27 @@ public class TeragrepSyntaxTests { "teragrep12", "teragrep_csv_schema", "teragrep_csv_header", - "teragrep_archive_summary" + "teragrep_archive_summary", + "teragrep_syslog_stream", + "teragrep_syslog_stream_host_port" }) - public void teragrepSyntaxParseTest(String arg) throws Exception { + public void teragrepSyntaxParseTest(String arg) { String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; ParserSyntaxTestingUtility parserSyntaxTestingUtility = new ParserSyntaxTestingUtility(fileName, false); - parserSyntaxTestingUtility.syntaxParseTest(arg); + Assertions.assertDoesNotThrow(() -> parserSyntaxTestingUtility.syntaxParseTest(arg)); } @ParameterizedTest @ValueSource(strings = { "teragrep", }) - void xpathTest1(String arg) throws Exception { + void xpathTest1(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/value"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -97,12 +100,12 @@ void xpathTest1(String arg) throws Exception { @ValueSource(strings = { "teragrep", }) - void xpathTest2(String arg) throws Exception { + void xpathTest2(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_getParameter/t_getTeragrepVersionParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -111,12 +114,12 @@ void xpathTest2(String arg) throws Exception { @ValueSource(strings = { "teragrep4", }) - void xpathTest3(String arg) throws Exception { + void xpathTest3(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_kafkaSaveModeParameter/value[1]"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -125,12 +128,12 @@ void xpathTest3(String arg) throws Exception { @ValueSource(strings = { "teragrep_tokenizer", }) - void xpathTestTokenizer(String arg) throws Exception { + void xpathTestTokenizer(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_tokenizerParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -139,12 +142,12 @@ void xpathTestTokenizer(String arg) throws Exception { @ValueSource(strings = { "teragrep7", }) - void xpathTestDynatrace(String arg) throws Exception { + void xpathTestDynatrace(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_dynatraceParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -153,12 +156,12 @@ void xpathTestDynatrace(String arg) throws Exception { @ValueSource(strings = { "teragrep8", }) - void xpathTestDynatraceWithOptionalParam(String arg) throws Exception { + void xpathTestDynatraceWithOptionalParam(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_dynatraceParameter/stringType"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -167,12 +170,12 @@ void xpathTestDynatraceWithOptionalParam(String arg) throws Exception { @ValueSource(strings = { "teragrep9", }) - void xpathTestCsvSave(String arg) throws Exception { + void xpathTestCsvSave(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -181,12 +184,12 @@ void xpathTestCsvSave(String arg) throws Exception { @ValueSource(strings = { "teragrep10", }) - void xpathTestJsonSave(String arg) throws Exception { + void xpathTestJsonSave(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -195,12 +198,12 @@ void xpathTestJsonSave(String arg) throws Exception { @ValueSource(strings = { "teragrep11", }) - void xpathTestDefaultSave(String arg) throws Exception { + void xpathTestDefaultSave(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_saveModeParameter/t_hdfsFormatParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -209,12 +212,12 @@ void xpathTestDefaultSave(String arg) throws Exception { @ValueSource(strings = { "teragrep12", }) - void xpathTestCsvLoad(String arg) throws Exception { + void xpathTestCsvLoad(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_hdfsFormatParameter"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -222,12 +225,12 @@ void xpathTestCsvLoad(String arg) throws Exception { @ValueSource(strings = { "teragrep_csv_header", }) - void xpathTestCsvLoadHeader(String arg) throws Exception { + void xpathTestCsvLoadHeader(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_headerParameter/booleanType"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -236,12 +239,12 @@ void xpathTestCsvLoadHeader(String arg) throws Exception { @ValueSource(strings = { "teragrep_csv_schema", }) - void xpathTestCsvLoadSchema(String arg) throws Exception { + void xpathTestCsvLoadSchema(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_execParameter/t_loadModeParameter/t_schemaParameter/stringType"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } @@ -250,13 +253,55 @@ void xpathTestCsvLoadSchema(String arg) throws Exception { @ValueSource(strings = { "teragrep_archive_summary", }) - void xpathTestArchiveSummary(String arg) throws Exception { + void xpathTestArchiveSummary(String arg) { ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; String xpathExp = "/root/transformStatement/teragrepTransformation/t_getParameter/t_getArchiveSummaryParameter/searchTransformationRoot/directoryStatement/directoryStatement/indexStatement/stringType"; - NodeList nodesA = (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false); + NodeList nodesA = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, xpathExp, false)); // Check that 1 found assertEquals(1,nodesA.getLength()); } + + @ParameterizedTest + @ValueSource(strings = { + "teragrep_syslog_stream", + }) + void syslogStreamTest(String arg) { // includes an eval command in the end of the query to make sure commands given after syslog work too + ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); + String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; + String syslogPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter"; + String evalPath = "/root/transformStatement/transformStatement/evalTransformation"; + + NodeList syslogNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, syslogPath, true)); + NodeList evalNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, evalPath, false)); + + // Check that 1 found + assertEquals(1,syslogNodes.getLength()); + assertEquals(1,evalNodes.getLength()); + } + + @ParameterizedTest + @ValueSource(strings = { + "teragrep_syslog_stream_host_port", + }) + void syslogStreamWithHostAndPortTest(String arg) { // includes an eval command in the end of the query to make sure commands given after syslog work too + ParserStructureTestingUtility pstu = new ParserStructureTestingUtility(); + String fileName = "src/test/resources/antlr4/commands/teragrep/" + arg + ".txt"; + String syslogPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter"; + String hostPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter/t_hostParameter"; + String portPath = "/root/transformStatement/teragrepTransformation/t_execParameter/t_syslogModeParameter/t_hostParameter/t_portParameter"; + String evalPath = "/root/transformStatement/transformStatement/evalTransformation"; + + NodeList syslogNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, syslogPath, false)); + NodeList hostNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, hostPath, false)); + NodeList portNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, portPath, false)); + NodeList evalNodes = Assertions.assertDoesNotThrow(() -> (NodeList) pstu.xpathQueryFile(fileName, evalPath, false)); + + // Check that 1 found + assertEquals(1,syslogNodes.getLength()); + assertEquals(1,hostNodes.getLength()); + assertEquals(1,portNodes.getLength()); + assertEquals(1,evalNodes.getLength()); + } } diff --git a/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt b/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt new file mode 100644 index 0000000..7419cfa --- /dev/null +++ b/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream.txt @@ -0,0 +1,46 @@ + +| teragrep exec syslog stream | eval a=1 \ No newline at end of file diff --git a/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream_host_port.txt b/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream_host_port.txt new file mode 100644 index 0000000..db519a6 --- /dev/null +++ b/src/test/resources/antlr4/commands/teragrep/teragrep_syslog_stream_host_port.txt @@ -0,0 +1,46 @@ + +| teragrep exec syslog stream host 12.12.123.123 port 123 | eval a=1 \ No newline at end of file