From f89c5205b38f054978b42263a71e3b0fa21afdcc Mon Sep 17 00:00:00 2001 From: David Gregory <2992938+DavidGregory084@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:00:20 +0100 Subject: [PATCH] Integrate ST error reporting with ANTLR's grammar error reporting Signed-off-by: David Gregory <2992938+DavidGregory084@users.noreply.github.com> --- .../v4/test/tool/TestActionTemplates.java | 163 +++++++++- tool/src/org/antlr/v4/tool/ErrorType.java | 32 +- .../v4/tool/GrammarTransformPipeline.java | 306 ++++++++++++++++-- 3 files changed, 472 insertions(+), 29 deletions(-) diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java index a77d6a9a3f..93d93ae358 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestActionTemplates.java @@ -27,11 +27,14 @@ public class TestActionTemplates { State state = execLexer(grammar, "34 34", tempDir, actionTemplates); assertInstanceOf(GeneratedState.class, state); + GeneratedState generated = (GeneratedState) state; + assertTrue(generated.containsErrors()); + assertEquals( "State: Generate; \n" + - "error(207): error reading action templates file " + actionTemplates + ": " + + "error(208): error reading action templates file " + actionTemplates + ": " + "Group file names must end in .stg: " + actionTemplates + "\n", generated.getErrorMessage()); } @@ -46,15 +49,144 @@ public class TestActionTemplates { State state = execLexer(grammar, "34 34", tempDir, actionTemplates); assertInstanceOf(GeneratedState.class, state); + GeneratedState generated = (GeneratedState) state; + assertTrue(generated.containsErrors()); + assertEquals( "State: Generate; \n" + "error(206): cannot find action templates file " + actionTemplates + " given for L\n", generated.getErrorMessage()); } - @Test void testActionTemplateLexerAction(@TempDir Path tempDir) { + @Test void testUnlexableActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {<¢>} ;\n"+ + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:16: invalid character '¢'\n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:14: this doesn't look like a template: \" <¢> \"\n", + generated.getErrorMessage()); + } + + @Test void testUnlexableMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {\n" + + " <¢>\n" + + "};\n"+ + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + assertInstanceOf(GeneratedState.class, state); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 3:3: invalid character '¢'\n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 3:0: mismatched input ' ' expecting EOF\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplateGroup(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) := <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {} ;\n"+ + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(210): error rendering action template file " + actionTemplates + ": Java.stg 1:11: mismatched input ':' expecting '::='\n" + + "error(212): " + grammarFile + ":2:14: error rendering action template: 2:16: no such template: /writeln\n", + generated.getErrorMessage()); + } + + @Test void testInvalidActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ { skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 2:29: premature EOF\n", + generated.getErrorMessage()); + } + + @Test void testInvalidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + String grammarFile = tempDir + FileSeparator + "L.g4"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {\n"+ + " skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + GeneratedState generated = (GeneratedState) state; + + assertTrue(generated.containsErrors()); + + assertEquals( + "State: Generate; \n" + + "error(211): " + grammarFile + ":2:14: error compiling action template: 4:1: premature EOF\n", + generated.getErrorMessage()); + } + + @Test void testValidActionTemplate(@TempDir Path tempDir) { writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); String actionTemplates = tempDir + FileSeparator + "Java.stg"; @@ -79,6 +211,33 @@ public class TestActionTemplates { assertEquals(expecting, ((ExecutedState) state).output); } + @Test void testValidMultilineActionTemplate(@TempDir Path tempDir) { + writeActionTemplatesFile(tempDir, "writeln(s) ::= <\");>>"); + + String actionTemplates = tempDir + FileSeparator + "Java.stg"; + + String grammar = + "lexer grammar L;\n"+ + "I : '0'..'9'+ {\n"+ + " \n"+ + "};\n"+ + "WS : (' '|'\\n') -> skip ;"; + + State state = execLexer(grammar, "34 34", tempDir, actionTemplates); + + // Should have identical output to TestLexerActions.testActionExecutedInDFA + String expecting = + "I\n" + + "I\n" + + "[@0,0:1='34',<1>,1:0]\n" + + "[@1,3:4='34',<1>,1:3]\n" + + "[@2,5:4='',<-1>,1:5]\n"; + + assertInstanceOf(ExecutedState.class, state); + + assertEquals(expecting, ((ExecutedState) state).output); + } + @Test void testActionTemplateHeader(@TempDir Path tempDir) { String actionTemplates = "normalizerImports() ::= <<\n" + diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index 20a569cd55..8ec88d61fb 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -1244,9 +1244,39 @@ public enum ErrorType { /** * Compiler Error 207. * + *

cannot find action tempaltes file filename

+ */ + CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR(207, "cannot find action templates file ", ErrorSeverity.ERROR), + /** + * Compiler Error 208. + * *

error reading action templates file filename: reason

*/ - ERROR_READING_ACTION_TEMPLATES_FILE(207, "error reading action templates file : ", ErrorSeverity.ERROR), + ERROR_READING_ACTION_TEMPLATES_FILE(208, "error reading action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 209. + * + *

error compiling action template file filename: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATE_FILE(209, "error compiling action templates file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 210. + * + *

error rendering action template file filename: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATE_FILE(210, "error rendering action template file : ", ErrorSeverity.ERROR), + /** + * Compiler Error 211. + * + *

error compiling action template: reason

+ */ + ERROR_COMPILING_ACTION_TEMPLATE(211, "error compiling action template: ", ErrorSeverity.ERROR), + /** + * Compiler Error 212. + * + *

error rendering action template: reason

+ */ + ERROR_RENDERING_ACTION_TEMPLATE(212, "error rendering action template: ", ErrorSeverity.ERROR), // Dependency sorting errors diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index 2dbb84d901..5a96c0e798 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -7,6 +7,7 @@ package org.antlr.v4.tool; import org.antlr.runtime.CommonToken; +import org.antlr.runtime.RecognitionException; import org.antlr.runtime.tree.CommonTree; import org.antlr.runtime.tree.CommonTreeNodeStream; import org.antlr.runtime.tree.Tree; @@ -27,11 +28,12 @@ import org.antlr.v4.tool.ast.GrammarRootAST; import org.antlr.v4.tool.ast.RuleAST; import org.antlr.v4.tool.ast.TerminalAST; -import org.stringtemplate.v4.ST; -import org.stringtemplate.v4.STGroupFile; -import org.stringtemplate.v4.STGroupString; +import org.stringtemplate.v4.*; import org.stringtemplate.v4.compiler.STException; +import org.stringtemplate.v4.misc.*; +import java.io.PrintWriter; +import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -517,16 +519,262 @@ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) { return lexerAST; } - public void expandActionTemplates(GrammarRootAST root) { - String fileName = root.g.getActionTemplates(); + /** + * Create an error listener for .stg action template group files provided via grammar options or via the command line. + */ + private STErrorListener createActionTemplateErrorListener(GrammarAST ast, STGroupFile actionTemplateGroupFile) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + @Override + public void compileTimeError(STMessage stMessage) { + reportActionTemplateError(stMessage); + } + + @Override + public void runTimeError(STMessage stMessage) { + reportActionTemplateError(stMessage); + } + + @Override + public void IOError(STMessage stMessage) { + reportInternalError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportInternalError(stMessage); + } + + private void reportActionTemplateError(STMessage stMessage) { + errorManager.toolError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE_FILE, + actionTemplateGroupFile.fileName, + stMessage.toString()); + } + + private void reportInternalError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + /** + * Create an error listener for action templates embedded inside a grammar's actions and semantic predicates. + */ + private STErrorListener createGrammarErrorListener(GrammarAST ast) { + return new STErrorListener() { + private final ErrorManager errorManager = ast.g.tool.errMgr; + + /** + * Get the STCompiletimeMesage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTCompiletimeErrorMessage(STCompiletimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + if (stMsg.token != null) { + int linePos = ast.getLine() + stMsg.token.getLine() - 1; + int charPos = stMsg.token.getCharPositionInLine(); + if (stMsg.token.getLine() == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2); + + pw.print(msg); + + return sw.toString(); + } + + /** + * Get the STLexerMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTLexerErrorMessage(STLexerMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + + // From STLexerMessage.toString + RecognitionException re = (RecognitionException)stMsg.cause; + int linePos = ast.getLine() + re.line - 1; + int charPos = re.charPositionInLine; + if (re.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + + String msg = String.format(stMsg.error.message, stMsg.msg); + + pw.print(msg); + + return sw.toString(); + } + + @Override + public void compileTimeError(STMessage stMessage) { + if (stMessage instanceof STCompiletimeMessage) { + STCompiletimeMessage compileMessage = (STCompiletimeMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTCompiletimeErrorMessage(compileMessage)); + } + else if (stMessage instanceof STLexerMessage) { + STLexerMessage lexerMessage = (STLexerMessage) stMessage; + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), getSTLexerErrorMessage(lexerMessage)); + } + else { + errorManager.grammarError(ErrorType.ERROR_COMPILING_ACTION_TEMPLATE, ast.g.fileName, ast.getToken(), stMessage.toString()); + } + } + + /** + * Get the STRuntimeMessage error location Coordinate. + */ + private Coordinate getSTRuntimeMessageSourceLocation(STRuntimeMessage msg) { + // From STRuntimeMessage.getSourceLocation + if (msg.ip >= 0 && msg.self != null && msg.self.impl != null) { + Interval I = msg.self.impl.sourceMap[msg.ip]; + if (I == null) { + return null; + } else { + int i = I.a; + return Misc.getLineCharPosition(msg.self.impl.template, i); + } + } else { + return null; + } + } + + /** + * Get the STRuntimeMessage error message content, translating the source location + * according to the action token's position in the grammar. + */ + private String getSTRuntimeErrorMessage(STRuntimeMessage stMsg) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + Coordinate coord = getSTRuntimeMessageSourceLocation(stMsg); + + if (coord != null) { + int linePos = ast.getLine() + coord.line - 1; + int charPos = coord.charPosition; + if (coord.line == 1) { + charPos += ast.getCharPositionInLine(); + } + pw.print(linePos + ":" + charPos + ": "); + } + + // From STMessage.toString + String msg = String.format(stMsg.error.message, stMsg.arg, stMsg.arg2, stMsg.arg3); + + pw.print(msg); + + if (stMsg.cause != null) { + pw.print("\nCaused by: "); + stMsg.cause.printStackTrace(pw); + } + + return sw.toString(); + } + + @Override + public void runTimeError(STMessage stMessage) { + if (stMessage instanceof STRuntimeMessage) { + STRuntimeMessage runtimeMessage = (STRuntimeMessage) stMessage; + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + getSTRuntimeErrorMessage(runtimeMessage)); + } else { + errorManager.grammarError( + ErrorType.ERROR_RENDERING_ACTION_TEMPLATE, + ast.g.fileName, + ast.getToken(), + stMessage.toString()); + } + } + + @Override + public void IOError(STMessage stMessage) { + reportError(stMessage); + } + + @Override + public void internalError(STMessage stMessage) { + reportError(stMessage); + } + + private void reportError(STMessage stMessage) { + errorManager.toolError(ErrorType.INTERNAL_ERROR, stMessage.cause, stMessage.toString()); + } + }; + } + + public STGroupFile loadActionTemplatesGroupFile(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + String actionTemplatesFile = grammar.getActionTemplates(); + try { - STGroupFile actionTemplates = new STGroupFile(fileName); - TreeVisitor v = new TreeVisitor(new GrammarASTAdaptor()); - v.visit(root, new TreeVisitorAction() { + STGroupFile actionTemplates = new STGroupFile(actionTemplatesFile); + STErrorListener errorListener = createActionTemplateErrorListener(root, actionTemplates); + + // Force load the action templates group file + actionTemplates.setListener(errorListener); + actionTemplates.load(); + + return actionTemplates; + + } catch (IllegalArgumentException e) { + if (e.getMessage() != null && e.getMessage().startsWith("No such group file")) { + // Check whether this action template file came from options in the grammar file + GrammarAST optionAST = root.getOptionAST("actionTemplates"); + + if (optionAST != null && actionTemplatesFile.equals(optionAST.getToken().getText())) { + errorManager.grammarError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_REFD_IN_GRAMMAR, + grammar.fileName, + optionAST.getToken(), actionTemplatesFile); + } else { + errorManager.toolError( + ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE, + actionTemplatesFile, + grammar.name); + } + + } else { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + } catch (STException e) { + errorManager.toolError( + ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, + actionTemplatesFile, + e.getMessage()); + } + + return null; + } + + public void expandActionTemplates(GrammarRootAST root) { + Grammar grammar = root.g; + ErrorManager errorManager = grammar.tool.errMgr; + + STGroupFile actionTemplates = loadActionTemplatesGroupFile(root); + + if (actionTemplates != null) { + TreeVisitor visitor = new TreeVisitor(new GrammarASTAdaptor()); + visitor.visit(root, new TreeVisitorAction() { @Override public Object pre(Object t) { - if (((GrammarAST) t).getType() == ANTLRParser.ACTION) { - return expandActionTemplate((GrammarAST) t, actionTemplates); + GrammarAST grammarAST = (GrammarAST) t; + int tokenType = grammarAST.getType(); + if (tokenType == ANTLRParser.ACTION || tokenType == ANTLRParser.SEMPRED) { + return expandActionTemplate((GrammarAST) t, errorManager, actionTemplates); } return t; } @@ -535,25 +783,31 @@ public Object post(Object t) { return t; } }); - } catch (IllegalArgumentException e) { - if (e.getMessage() != null && e.getMessage().startsWith("No such group file")) { - tool.errMgr.toolError(ErrorType.CANNOT_FIND_ACTION_TEMPLATES_FILE_GIVEN_ON_CMDLINE, e, fileName, root.g.name); - } else { - tool.errMgr.toolError(ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, fileName, e.getMessage()); - } - } catch (STException e) { - tool.errMgr.toolError(ErrorType.ERROR_READING_ACTION_TEMPLATES_FILE, e, fileName, e.getMessage()); } } - public GrammarAST expandActionTemplate(GrammarAST t, STGroupFile actionTemplates) { - String grammarFileInfo = t.g.fileName + " " + t.getLine() + 1 + ":" + t.getCharPositionInLine(); - String actionText = t.getText().substring(1, t.getText().length() - 1); - STGroupString actionTemplateGroup = new STGroupString(grammarFileInfo, "action() ::= << " + actionText + " >>"); - actionTemplateGroup.importTemplates(actionTemplates); + public GrammarAST expandActionTemplate(GrammarAST ast, ErrorManager errMgr, STGroupFile actionTemplateGroupFile) { + // Trim the curly braces and trailing question mark + // from an action or semantic predicate + String actionText = ast.getText() + .substring(1, + ast.getType() == ANTLRParser.SEMPRED + ? ast.getText().length() - 2 + : ast.getText().length() - 1); + + STGroupString actionTemplateGroup = new STGroupString( + ast.g.fileName, "action() ::= << " + actionText + " >>"); + + actionTemplateGroup.importTemplates(actionTemplateGroupFile); + + actionTemplateGroup.setListener(createGrammarErrorListener(ast)); + ST actionTemplate = actionTemplateGroup.getInstanceOf("action"); - String expandedTemplate = actionTemplate.render(); - t.setText(expandedTemplate); - return t; + + if (actionTemplate != null) { + ast.setText(actionTemplate.render()); + } + + return ast; } }