[23] Formatter support for markdown doc comments

Restore ability to format Java in file with markdown comments:
+ avoid AssertionError in TokenManager.firstIndexIn
  visit(Javadoc) need to look for TokenNameCOMMENT_MARKDOWN specifically
+ fix infinite loop in tokenizeMultilineComment()
  - detect markdown comment
  - search for '/' not '*'
  - token should not include the final '\n'

Leave markdown comments unchanged by formatting for now

related to eclipse-jdt#2851

org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/CommentsPreparator.java

@@ -20,6 +20,7 @@
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE;
+import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameTextBlock;
@@ -587,7 +588,7 @@ public boolean visit(Javadoc node) {
 		this.commonAttributeAnnotations.clear();
 		this.ctm = null;
 
-		int commentIndex = this.tm.firstIndexIn(node, TokenNameCOMMENT_JAVADOC);
+		int commentIndex = this.tm.firstIndexIn(node, node.isMarkdown() ? TokenNameCOMMENT_MARKDOWN : TokenNameCOMMENT_JAVADOC);
 		Token commentToken = this.tm.get(commentIndex);
 
 		if (node.getParent() == null) {
@@ -1209,7 +1210,8 @@ private boolean tokenizeMultilineComment(Token commentToken) {
 		if (this.allowSubstituteWrapping == null || this.allowSubstituteWrapping.length < commentToken.countChars()) {
 			this.allowSubstituteWrapping = new boolean[commentToken.countChars()];
 		}
-		boolean isJavadoc = commentToken.tokenType == TokenNameCOMMENT_JAVADOC;
+		boolean isMarkdown = commentToken.tokenType == TokenNameCOMMENT_MARKDOWN;
+		boolean isJavadoc = commentToken.tokenType == TokenNameCOMMENT_JAVADOC || isMarkdown;
 		Arrays.fill(this.allowSubstituteWrapping, 0, commentToken.countChars(), !isJavadoc);
 
 		final boolean cleanBlankLines = isJavadoc ? this.options.comment_clear_blank_lines_in_javadoc_comment
@@ -1218,14 +1220,15 @@ private boolean tokenizeMultilineComment(Token commentToken) {
 		List<Token> structure = new ArrayList<>();
 
 		int firstTokenEnd = commentToken.originalStart + 1;
-		while (firstTokenEnd < commentToken.originalEnd - 1 && this.tm.charAt(firstTokenEnd + 1) == '*')
+		char markerChar = isMarkdown ? '/' : '*';
+		while (firstTokenEnd < commentToken.originalEnd - 1 && this.tm.charAt(firstTokenEnd + 1) == markerChar)
 			firstTokenEnd++;
 		Token first = new Token(commentToken.originalStart, firstTokenEnd, commentToken.tokenType);
 		first.spaceAfter();
 		structure.add(first);
 
 		int lastTokenStart = commentToken.originalEnd - 1;
-		while (lastTokenStart - 1 > firstTokenEnd && this.tm.charAt(lastTokenStart - 1) == '*')
+		while (lastTokenStart - 1 > firstTokenEnd && this.tm.charAt(lastTokenStart - 1) == markerChar)
 			lastTokenStart--;
 
 		int position = firstTokenEnd + 1;
@@ -1241,7 +1244,7 @@ private boolean tokenizeMultilineComment(Token commentToken) {
 						i++;
 					position = i + 1;
 				} else if (!ScannerHelper.isWhitespace(c)) {
-					while (this.tm.charAt(i) == '*' && lineBreaks > 0)
+					while (this.tm.charAt(i) == markerChar && lineBreaks > 0)
 						i++;
 					position = i;
 					break;

org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/TextEditsBuilder.java

@@ -16,6 +16,7 @@
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC;
+import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameNotAToken;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameStringLiteral;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameTextBlock;
@@ -121,6 +122,12 @@ protected boolean token(Token token, int index) {
 
 		bufferWhitespaceBefore(token, index);
 
+		if (token.tokenType == TokenNameCOMMENT_MARKDOWN) {
+			flushBuffer(token.originalStart);
+			this.counter = token.originalEnd + 1;
+			return true; // don't touch markdown format for now.
+		}
+
 		List<Token> structure = token.getInternalStructure();
 		if (token.tokenType == TokenNameCOMMENT_LINE) {
 			handleSingleLineComment(token, index);

org.eclipse.jdt.core/formatter/org/eclipse/jdt/internal/formatter/Token.java

@@ -17,6 +17,7 @@
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_BLOCK;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_JAVADOC;
 import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_LINE;
+import static org.eclipse.jdt.internal.compiler.parser.TerminalTokens.TokenNameCOMMENT_MARKDOWN;
 
 import java.util.List;
 
@@ -142,7 +143,7 @@ public Token(Token tokenToCopy, int newOriginalStart, int newOriginalEnd, int ne
 	public static Token fromCurrent(Scanner scanner, int currentToken) {
 		int start = scanner.getCurrentTokenStartPosition();
 		int end = scanner.getCurrentTokenEndPosition();
-		if (currentToken == TokenNameCOMMENT_LINE) {
+		if (currentToken == TokenNameCOMMENT_LINE || currentToken == TokenNameCOMMENT_MARKDOWN) {
 			// don't include line separator, but set break-after
 			while (end > start) {
 				char c = scanner.source[end];