Proposal: Add action templates functionality

runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java

@@ -24,14 +24,15 @@ public class RunOptions {
 	public final boolean showDFA;
 	public final Stage endStage;
 	public final String superClass;
+	public final String actionTemplates;
 	public final PredictionMode predictionMode;
 	public final boolean buildParseTree;
 
 	public RunOptions(String grammarFileName, String grammarStr, String parserName, String lexerName,
 					  boolean useListener, boolean useVisitor, String startRuleName,
 					  String input, boolean profile, boolean showDiagnosticErrors,
 					  boolean traceATN, boolean showDFA, Stage endStage,
-					  String language, String superClass, PredictionMode predictionMode, boolean buildParseTree) {
+					  String language, String superClass, String actionTemplates, PredictionMode predictionMode, boolean buildParseTree) {
 		this.grammarFileName = grammarFileName;
 		this.grammarStr = grammarStr;
 		this.parserName = parserName;
@@ -69,6 +70,7 @@ else if (lexerName != null) {
 		this.showDFA = showDFA;
 		this.endStage = endStage;
 		this.superClass = superClass;
+		this.actionTemplates = actionTemplates;
 		this.predictionMode = predictionMode;
 		this.buildParseTree = buildParseTree;
 	}

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java

@@ -172,6 +172,10 @@ public State run(RunOptions runOptions) {
 			options.add("-DsuperClass=" + runOptions.superClass);
 		}
 
+		if (runOptions.actionTemplates != null && runOptions.actionTemplates.length() > 0) {
+			options.add("-DactionTemplates=" + runOptions.actionTemplates);
+		}
+
 		// See if the target wants to add tool options.
 		//
 		List<String> targetOpts = getTargetToolOptions(runOptions);

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java

@@ -167,6 +167,7 @@ private static String test(RuntimeTestDescriptor descriptor, RuntimeRunner runne
 				Stage.Execute,
 				targetName,
 				superClass,
+				null,
 				descriptor.predictionMode,
 				descriptor.buildParseTree
 		);

runtime-testsuite/test/org/antlr/v4/test/runtime/TraceATN.java

@@ -158,6 +158,7 @@ public String test(RuntimeTestDescriptor descriptor, RuntimeRunner runner, Strin
 				Stage.Execute,
 				targetName,
 				superClass,
+				null,
 				PredictionMode.LL,
 				true
 		);

tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java

@@ -0,0 +1,131 @@
+package org.antlr.v4.test.tool;
+
+import org.antlr.v4.test.runtime.RunOptions;
+import org.antlr.v4.test.runtime.Stage;
+import org.antlr.v4.test.runtime.java.JavaRunner;
+import org.antlr.v4.test.runtime.states.ExecutedState;
+import org.antlr.v4.test.runtime.states.GeneratedState;
+import org.antlr.v4.test.runtime.states.State;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+
+import static org.antlr.v4.test.runtime.FileUtils.writeFile;
+import static org.antlr.v4.test.runtime.RuntimeTestUtils.FileSeparator;
+import static org.antlr.v4.test.tool.ToolTestUtils.createOptionsForJavaToolTests;
+import static org.junit.jupiter.api.Assertions.*;
+
+public class TestActionTemplates {
+	@Test() void testIncorrectActionTemplateGroupExtension(@TempDir Path tempDir) {
+		String actionTemplates = tempDir + FileSeparator + "Java.st";
+
+		String grammar =
+			"lexer grammar L;"+
+			"WS : (' '|'\\n') -> skip ;";
+
+		State state = execLexer(grammar, "34 34", tempDir, actionTemplates);
+
+		assertInstanceOf(GeneratedState.class, state);
+		GeneratedState generated = (GeneratedState) state;
+		assertTrue(generated.containsErrors());
+		assertEquals(
+			"State: Generate; \n" +
+			"error(207):  error reading action templates file " + actionTemplates + ": " +
+			"Group file names must end in .stg: " + actionTemplates + "\n",
+			generated.getErrorMessage());
+	}
+
+	@Test() void testActionTemplateFileMissing(@TempDir Path tempDir) {
+		String actionTemplates = tempDir + FileSeparator + "Java.stg";
+
+		String grammar =
+			"lexer grammar L;"+
+			"WS : (' '|'\\n') -> skip ;";
+
+		State state = execLexer(grammar, "34 34", tempDir, actionTemplates);
+
+		assertInstanceOf(GeneratedState.class, state);
+		GeneratedState generated = (GeneratedState) state;
+		assertTrue(generated.containsErrors());
+		assertEquals(
+			"State: Generate; \n" +
+			"error(206):  cannot find action templates file " + actionTemplates + " given for L\n",
+			generated.getErrorMessage());
+	}
+
+	@Test void testActionTemplateLexerAction(@TempDir Path tempDir) {
+		writeActionTemplatesFile(tempDir, "writeln(s) ::= <<outStream.println(\"<s>\");>>");
+
+		String actionTemplates = tempDir + FileSeparator + "Java.stg";
+
+		String grammar =
+			"lexer grammar L;\n"+
+			"I : '0'..'9'+ {<writeln(\"I\")>} ;\n"+
+			"WS : (' '|'\\n') -> skip ;";
+
+		State state = execLexer(grammar, "34 34", tempDir, actionTemplates);
+
+		// Should have identical output to TestLexerActions.testActionExecutedInDFA
+		String expecting =
+			"I\n" +
+			"I\n" +
+			"[@0,0:1='34',<1>,1:0]\n" +
+			"[@1,3:4='34',<1>,1:3]\n" +
+			"[@2,5:4='<EOF>',<-1>,1:5]\n";
+
+		assertInstanceOf(ExecutedState.class, state);
+
+		assertEquals(expecting, ((ExecutedState) state).output);
+	}
+
+	@Test void testActionTemplateHeader(@TempDir Path tempDir) {
+		String actionTemplates =
+			"normalizerImports() ::= <<\n" +
+			"import java.text.Normalizer;\n" +
+			"import java.text.Normalizer.Form;\n" +
+			">>\n" +
+			"normalize(s) ::= <<Normalizer.normalize(<s>, Form.NFKC)>>\n" +
+			"getText() ::= <<getText()>>\n" +
+			"setText(s) ::= <<setText(<s>);>>";
+
+		writeActionTemplatesFile(tempDir, actionTemplates);
+
+		String actionTemplatesFile = tempDir + FileSeparator + "Java.stg";
+
+		String grammar =
+			"lexer grammar L;\n"+
+			"@lexer::header {\n"+
+			"<normalizerImports()>\n"+
+			"}\n"+
+			"ID : (ID_START ID_CONTINUE* | '_' ID_CONTINUE+) { <setText(normalize(getText()))> } ;\n"+
+			"ID_START : [\\p{XID_Start}] ;\n"+
+			"ID_CONTINUE: [\\p{XID_Continue}] ;\n"+
+			"WS : (' '|'\\n') -> skip ;";
+
+		State state = execLexer(grammar, "This _is \ufb01ne", tempDir, actionTemplatesFile);
+
+		String expecting =
+			"[@0,0:3='This',<1>,1:0]\n"+
+			"[@1,5:7='_is',<1>,1:5]\n"+
+			"[@2,9:11='fine',<1>,1:9]\n"+
+			"[@3,12:11='<EOF>',<-1>,1:12]\n";
+
+		assertInstanceOf(ExecutedState.class, state);
+
+		assertEquals(expecting, ((ExecutedState) state).output);
+	}
+
+	void writeActionTemplatesFile(Path tempDir, String template) {
+		writeFile(tempDir.toString(), "Java.stg", template);
+	}
+
+	State execLexer(String grammarStr, String input, Path tempDir, String actionTemplates) {
+		RunOptions runOptions = createOptionsForJavaToolTests("L.g4", grammarStr, null, "L",
+			false, true, null, input,
+			false, false, Stage.Execute, actionTemplates);
+		try (JavaRunner runner = new JavaRunner(tempDir, false)) {
+			return runner.run(runOptions);
+		}
+	}
+}

tool-testsuite/test/org/antlr/v4/test/tool/TestCompositeGrammars.java

@@ -727,7 +727,7 @@ private static boolean compile(String grammarFileName, String grammarStr, String
 	) {
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammarStr, parserName, null,
 				false, false, startRuleName, null,
-				false, false, Stage.Compile);
+				false, false, Stage.Compile, null);
 		try (JavaRunner runner = new JavaRunner(tempDirPath, false)) {
 			JavaCompiledState compiledState = (JavaCompiledState) runner.run(runOptions);
 			return !compiledState.containsErrors();

tool-testsuite/test/org/antlr/v4/test/tool/TestParseTreeMatcher.java

@@ -392,7 +392,7 @@ private static ParseTreeMatch checkPatternMatch(String grammar, String startRule
 		String lexerName = grammarName+"Lexer";
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName,
 				false, false, startRule, input,
-				false, false, Stage.Execute);
+				false, false, Stage.Execute, null);
 		try (JavaRunner runner = new JavaRunner()) {
 			JavaExecutedState executedState = (JavaExecutedState)runner.run(runOptions);
 			JavaCompiledState compiledState = (JavaCompiledState)executedState.previousState;
@@ -413,7 +413,7 @@ private static ParseTreePatternMatcher getPatternMatcher(
 	) throws Exception {
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName,
 				false, false, startRule, null,
-				false, false, Stage.Compile);
+				false, false, Stage.Compile, null);
 		try (JavaRunner runner = new JavaRunner()) {
 			JavaCompiledState compiledState = (JavaCompiledState) runner.run(runOptions);
 

tool-testsuite/test/org/antlr/v4/test/tool/TestParserProfiler.java

@@ -223,7 +223,7 @@ public class TestParserProfiler {
 
 		RunOptions runOptions = createOptionsForJavaToolTests("T.g4", grammar, "TParser", "TLexer",
 				false, false, "s", "xyz;abc;z.q",
-				true, false, Stage.Execute);
+				true, false, Stage.Execute, null);
 		try (JavaRunner runner = new JavaRunner()) {
 			ExecutedState state = (ExecutedState) runner.run(runOptions);
 			String expecting =

tool-testsuite/test/org/antlr/v4/test/tool/TestPerformance.java

@@ -1103,7 +1103,7 @@ protected JavaCompiledState compileJavaParser(boolean leftRecursive) throws IOEx
 
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, body, parserName, lexerName,
 				false, true, null, null,
-				false, false, Stage.Compile);
+				false, false, Stage.Compile, null);
 		try (RuntimeRunner runner = new JavaRunner()) {
 			return (JavaCompiledState) runner.run(runOptions);
 		}

tool-testsuite/test/org/antlr/v4/test/tool/TestXPath.java

@@ -201,7 +201,7 @@ private Pair<String[], Collection<ParseTree>> compileAndExtract(String grammarFi
 	) throws Exception {
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammar, parserName, lexerName,
 				false, false, startRuleName, input,
-				false, false, Stage.Execute);
+				false, false, Stage.Execute, null);
 		try (JavaRunner runner = new JavaRunner()) {
 			JavaExecutedState executedState = (JavaExecutedState)runner.run(runOptions);
 			JavaCompiledState compiledState = (JavaCompiledState)executedState.previousState;

tool-testsuite/test/org/antlr/v4/test/tool/ToolTestUtils.java

@@ -69,7 +69,7 @@ private static ExecutedState execRecognizer(String grammarFileName, String gramm
 										 Path workingDir, boolean saveTestDir) {
 		RunOptions runOptions = createOptionsForJavaToolTests(grammarFileName, grammarStr, parserName, lexerName,
 				false, true, startRuleName, input,
-				false, showDiagnosticErrors, Stage.Execute);
+				false, showDiagnosticErrors, Stage.Execute, null);
 		try (JavaRunner runner = new JavaRunner(workingDir, saveTestDir)) {
 			State result = runner.run(runOptions);
 			if (!(result instanceof ExecutedState)) {
@@ -83,11 +83,11 @@ public static RunOptions createOptionsForJavaToolTests(
 			String grammarFileName, String grammarStr, String parserName, String lexerName,
 			boolean useListener, boolean useVisitor, String startRuleName,
 			String input, boolean profile, boolean showDiagnosticErrors,
-			Stage endStage
+			Stage endStage, String actionTemplates
 	) {
 		return new RunOptions(grammarFileName, grammarStr, parserName, lexerName, useListener, useVisitor, startRuleName,
 				input, profile, showDiagnosticErrors, false, false, endStage, "Java",
-				JavaRunner.runtimeTestParserName, PredictionMode.LL, true);
+				JavaRunner.runtimeTestParserName, actionTemplates, PredictionMode.LL, true);
 	}
 
 	public static void testErrors(String[] pairs, boolean printTree) {

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -1235,6 +1235,18 @@ public enum ErrorType {
 	 */
 	@Deprecated
 	V3_SYNPRED(205, "(...)=> syntactic predicates are not supported in ANTLR 4", ErrorSeverity.ERROR),
+	/**
+	 * Compiler Error 206.
+	 *
+	 * <p>cannot find action templates file <em>filename</em></p>
+	 */
+	CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE(206, "cannot find action templates file <arg> given for <arg2>", ErrorSeverity.ERROR),
+	/**
+	 * Compiler Error 207.
+	 *
+	 * <p>error reading action templates file <em>filename</em>: <em>reason</em></p>
+	 */
+	ERROR_READING_ACTION_TEMPLATES_FILE(207, "error reading action templates file <arg>: <arg2>", ErrorSeverity.ERROR),
 
     // Dependency sorting errors
 

tool/src/org/antlr/v4/tool/Grammar.java

@@ -80,6 +80,7 @@ public class Grammar implements AttributeResolver {
 		parserOptions.add("TokenLabelType");
 		parserOptions.add("tokenVocab");
 		parserOptions.add("language");
+		parserOptions.add("actionTemplates");
 		parserOptions.add("accessLevel");
 		parserOptions.add("exportMacro");
 		parserOptions.add(caseInsensitiveOptionName);
@@ -1178,6 +1179,10 @@ public String getLanguage() {
 		return getOptionString("language");
 	}
 
+	public String getActionTemplates() {
+		return getOptionString("actionTemplates");
+	}
+
 	public String getOptionString(String key) { return ast.getOptionString(key); }
 
 	/** Given ^(TOKEN_REF ^(OPTIONS ^(ELEMENT_OPTIONS (= assoc right))))

tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java

@@ -27,6 +27,10 @@
 import org.antlr.v4.tool.ast.GrammarRootAST;
 import org.antlr.v4.tool.ast.RuleAST;
 import org.antlr.v4.tool.ast.TerminalAST;
+import org.stringtemplate.v4.ST;
+import org.stringtemplate.v4.STGroupFile;
+import org.stringtemplate.v4.STGroupString;
+import org.stringtemplate.v4.compiler.STException;
 
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -54,6 +58,10 @@ public void process() {
 		reduceBlocksToSets(root);
         expandParameterizedLoops(root);
 
+		if (g.getActionTemplates() != null)	 {
+			expandActionTemplates(root);
+		}
+
         tool.log("grammar", "after: "+root.toStringTree());
 	}
 
@@ -509,4 +517,43 @@ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) {
 		return lexerAST;
 	}
 
+	public void expandActionTemplates(GrammarRootAST root) {
+		String fileName = root.g.getActionTemplates();
+		try {
+			STGroupFile actionTemplates = new STGroupFile(fileName);
+			TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor());
+			v.visit(root, new TreeVisitorAction() {
+				@Override
+				public Object pre(Object t) {
+					if (((GrammarAST) t).getType() == ANTLRParser.ACTION) {
+						return expandActionTemplate((GrammarAST) t, actionTemplates);
+					}
+					return t;
+				}
+				@Override
+				public Object post(Object t) {
+					return t;
+				}
+			});
+		} catch (IllegalArgumentException e) {
+			if (e.getMessage() != null && e.getMessage().startsWith("No such group file")) {
+				tool.errMgr.toolError(ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE, e, fileName, root.g.name);
+			} else {
+				tool.errMgr.toolError(ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, fileName, e.getMessage());
+			}
+		} catch (STException e) {
+			tool.errMgr.toolError(ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, fileName, e.getMessage());
+		}
+	}
+
+	public GrammarAST expandActionTemplate(GrammarAST t, STGroupFile actionTemplates) {
+		String grammarFileInfo = t.g.fileName + " " + t.getLine() + 1 + ":" + t.getCharPositionInLine();
+		String actionText = t.getText().substring(1, t.getText().length() - 1);
+		STGroupString actionTemplateGroup = new STGroupString(grammarFileInfo, "action() ::= << " + actionText + " >>");
+		actionTemplateGroup.importTemplates(actionTemplates);
+		ST actionTemplate = actionTemplateGroup.getInstanceOf("action");
+		String expandedTemplate = actionTemplate.render();
+		t.setText(expandedTemplate);
+		return t;
+	}
 }