From cbce8dd0dc8c652e8543c7d17489e79c6af05f98 Mon Sep 17 00:00:00 2001 From: Stephan Herrmann Date: Sat, 24 Aug 2024 16:02:27 +0200 Subject: [PATCH] [23] Formatter support for markdown doc comments Restore ability to format Java in file with markdown comments: + avoid AssertionError in TokenManager.firstIndexIn visit(Javadoc) need to look for TokenNameCOMMENT_MARKDOWN specifically + fix infinite loop in tokenizeMultilineComment() - detect markdown comment - search for '/' not '*' - token should not include the final '\n' Leave markdown comments unchanged by formatting for now related to https://github.com/eclipse-jdt/eclipse.jdt.core/issues/2851 --- .../jdt/internal/formatter/CommentsPreparator.java | 13 ++++++++----- .../jdt/internal/formatter/TextEditsBuilder.java | 7 +++++++ .../org/eclipse/jdt/internal/formatter/Token.java | 3 ++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/CommentsPreparator.java b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/CommentsPreparator.java index a6003d06695..10aa3e1bbe5 100644 --- a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/CommentsPreparator.java +++ b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/CommentsPreparator.java @@ -20,6 +20,7 @@ import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE; +import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameTextBlock; @@ -587,7 +588,7 @@ public boolean visit(Javadoc node) { this.commonAttributeAnnotations.clear(); this.ctm = null; - int commentIndex = this.tm.firstIndexIn(node, TokenNameCOMMENT_JAVADOC); + int commentIndex = this.tm.firstIndexIn(node, node.isMarkdown() ? TokenNameCOMMENT_MARKDOWN : TokenNameCOMMENT_JAVADOC); Token commentToken = this.tm.get(commentIndex); if (node.getParent() == null) { @@ -1209,7 +1210,8 @@ private boolean tokenizeMultilineComment(Token commentToken) { if (this.allowSubstituteWrapping == null || this.allowSubstituteWrapping.length < commentToken.countChars()) { this.allowSubstituteWrapping = new boolean[commentToken.countChars()]; } - boolean isJavadoc = commentToken.tokenType == TokenNameCOMMENT_JAVADOC; + boolean isMarkdown = commentToken.tokenType == TokenNameCOMMENT_MARKDOWN; + boolean isJavadoc = commentToken.tokenType == TokenNameCOMMENT_JAVADOC || isMarkdown; Arrays.fill(this.allowSubstituteWrapping, 0, commentToken.countChars(), !isJavadoc); final boolean cleanBlankLines = isJavadoc ? this.options.comment_clear_blank_lines_in_javadoc_comment @@ -1218,14 +1220,15 @@ private boolean tokenizeMultilineComment(Token commentToken) { List structure = new ArrayList<>(); int firstTokenEnd = commentToken.originalStart + 1; - while (firstTokenEnd < commentToken.originalEnd - 1 && this.tm.charAt(firstTokenEnd + 1) == '*') + char markerChar = isMarkdown ? '/' : '*'; + while (firstTokenEnd < commentToken.originalEnd - 1 && this.tm.charAt(firstTokenEnd + 1) == markerChar) firstTokenEnd++; Token first = new Token(commentToken.originalStart, firstTokenEnd, commentToken.tokenType); first.spaceAfter(); structure.add(first); int lastTokenStart = commentToken.originalEnd - 1; - while (lastTokenStart - 1 > firstTokenEnd && this.tm.charAt(lastTokenStart - 1) == '*') + while (lastTokenStart - 1 > firstTokenEnd && this.tm.charAt(lastTokenStart - 1) == markerChar) lastTokenStart--; int position = firstTokenEnd + 1; @@ -1241,7 +1244,7 @@ private boolean tokenizeMultilineComment(Token commentToken) { i++; position = i + 1; } else if (!ScannerHelper.isWhitespace(c)) { - while (this.tm.charAt(i) == '*' && lineBreaks > 0) + while (this.tm.charAt(i) == markerChar && lineBreaks > 0) i++; position = i; break; diff --git a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/TextEditsBuilder.java b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/TextEditsBuilder.java index 34b7864ee26..0c5a2e82876 100644 --- a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/TextEditsBuilder.java +++ b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/TextEditsBuilder.java @@ -16,6 +16,7 @@ import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC; +import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameTextBlock; @@ -121,6 +122,12 @@ protected boolean token(Token token, int index) { bufferWhitespaceBefore(token, index); + if (token.tokenType == TokenNameCOMMENT_MARKDOWN) { + flushBuffer(token.originalStart); + this.counter = token.originalEnd + 1; + return true; // don't touch markdown format for now. + } + List structure = token.getInternalStructure(); if (token.tokenType == TokenNameCOMMENT_LINE) { handleSingleLineComment(token, index); diff --git a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/Token.java b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/Token.java index ca4d701f382..638cd1010bc 100644 --- a/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/Token.java +++ b/org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/Token.java @@ -17,6 +17,7 @@ import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC; import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE; +import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN; import java.util.List; @@ -142,7 +143,7 @@ public Token(Token tokenToCopy, int newOriginalStart, int newOriginalEnd, int ne public static Token fromCurrent(Scanner scanner, int currentToken) { int start = scanner.getCurrentTokenStartPosition(); int end = scanner.getCurrentTokenEndPosition(); - if (currentToken == TokenNameCOMMENT_LINE) { + if (currentToken == TokenNameCOMMENT_LINE || currentToken == TokenNameCOMMENT_MARKDOWN) { // don't include line separator, but set break-after while (end > start) { char c = scanner.source[end];