Skip to content

Commit

Permalink
Add improved syntax highlighting grammar (#317)
Browse files Browse the repository at this point in the history
This now pulls in a grammar for from:
<https://github.com/wooorm/markdown-tm-language>.

It fixes a bunch of previous errors and adds real support for MDX.
It also supports YAML frontmatter, TOML frontmatter, GFM (autolink
literals, footnotes, strikethrough, tables, tasklists), GitHub (gemoji,
mentions, references).
There’s support for about 20 common embedded grammars in fenced code
blocks.
Embedded code (in fenced code blocks, or in ESM/expressions) is now
marked as being the correct language, which makes comments and such
work.

Closes GH-183.
Closes GH-191.
Closes GH-209.
Closes GH-221.

Co-authored-by: Remco Haszing <[email protected]>
  • Loading branch information
wooorm and remcohaszing authored Mar 23, 2023
1 parent 7ae33ef commit f1bd49b
Show file tree
Hide file tree
Showing 12 changed files with 6,936 additions and 131 deletions.
18 changes: 18 additions & 0 deletions .changeset/small-tomatoes-act.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
---
'vscode-mdx': minor
---

Add improved syntax highlighting grammar

This now pulls in a grammar for from:
<https://github.com/wooorm/markdown-tm-language>.

It fixes a bunch of previous errors and adds real support for MDX.
It also supports YAML frontmatter, TOML frontmatter, GFM (autolink
literals, footnotes, strikethrough, tables, tasklists), GitHub (gemoji,
mentions, references).
There’s support for about 20 common embedded grammars in fenced code
blocks.
Embedded code (in fenced code blocks, or in ESM/expressions) is now
marked as being the correct language, which makes comments and such
work.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
*.d.ts
*.d.mts
*.log
*.tgz
*.tsbuildinfo
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"packages/*"
],
"scripts": {
"generate": "npm --workspace vscode-mdx run generate",
"build": "npm --workspaces run build",
"format": "remark . -qfo && prettier . -w --loglevel warn && xo --fix",
"start": "npm --workspace @mdx-js/monaco-demo run start",
Expand Down
2 changes: 2 additions & 0 deletions packages/language-server/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
{
"include": ["**/**.js"],
"exclude": ["coverage/", "node_modules/"],
"compilerOptions": {
"checkJs": true,
"module": "node16",
Expand Down
2 changes: 2 additions & 0 deletions packages/language-service/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
{
"include": ["**/**.js"],
"exclude": ["coverage/", "node_modules/"],
"compilerOptions": {
"checkJs": true,
"declaration": true,
Expand Down
2 changes: 2 additions & 0 deletions packages/monaco/tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
{
"include": ["**/**.js"],
"exclude": ["coverage/", "node_modules/"],
"compilerOptions": {
"checkJs": true,
"declaration": true,
Expand Down
1 change: 1 addition & 0 deletions packages/vscode-mdx/.vscodeignore
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,4 @@ demo/
fixtures/
node_modules/
packages/
script/
93 changes: 45 additions & 48 deletions packages/vscode-mdx/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,17 @@
"build": "esbuild extension=./src/extension.js language-server=@mdx-js/language-server --bundle --platform=node --target=node16 --external:vscode --outdir=out",
"build:debug": "npm run copy-libs && npm run build -- --sourcemap",
"copy-libs": "cpy '../../node_modules/typescript/lib/lib.*.d.ts' out/",
"generate": "node --conditions development ./script/build.mjs",
"vscode:prepublish": "npm run copy-libs && npm run build -- --minify"
},
"devDependencies": {
"@types/node": "^18.0.0",
"@types/vscode": "^1.0.0",
"@vscode/vsce": "^2.0.0",
"cpy-cli": "^4.0.0",
"esbuild": "^0.16.0",
"ovsx": "^0.8.0",
"undici": "^5.0.0",
"vscode-languageclient": "^8.0.0"
},
"contributes": {
Expand Down Expand Up @@ -86,55 +89,49 @@
"grammars": [
{
"language": "mdx",
"scopeName": "text.html.markdown.jsx",
"path": "./syntaxes/mdx.tmLanguage.json",
"scopeName": "source.mdx",
"path": "./syntaxes/source.mdx.tmLanguage",
"#1": "note: the below list maps TextMate scopes to VS Code Language IDs (<https://code.visualstudio.com/docs/languages/identifiers>), which does not affect syntax highlighting, but affects for example how comments work.",
"#2": "note: `graphql`, `kotlin`, `toml` have no names in that list, but do work with the most popular extension.",
"#3": "note: we map to JSX-enabled flavors of `js` and `ts`",
"#4": "note: I have no clue how to test `vbnet` (visual basic), I cannot find extensions",
"embeddedLanguages": {
"source.js": "javascript",
"source.js.jsx": "javascriptreact",
"meta.tag.js": "jsx-tags",
"meta.tag.without-attributes.js": "jsx-tags",
"meta.embedded.block.frontmatter": "yaml",
"meta.embedded.block.css": "css",
"meta.embedded.block.ini": "ini",
"meta.embedded.block.java": "java",
"meta.embedded.block.lua": "lua",
"meta.embedded.block.makefile": "makefile",
"meta.embedded.block.perl": "perl",
"meta.embedded.block.r": "r",
"meta.embedded.block.ruby": "ruby",
"meta.embedded.block.php": "php",
"meta.embedded.block.sql": "sql",
"meta.embedded.block.vs_net": "vs_net",
"meta.embedded.block.html": "html",
"meta.embedded.block.xml": "xml",
"meta.embedded.block.xsl": "xsl",
"meta.embedded.block.yaml": "yaml",
"meta.embedded.block.dosbatch": "dosbatch",
"meta.embedded.block.clojure": "clojure",
"meta.embedded.block.coffee": "coffee",
"meta.embedded.block.c": "c",
"meta.embedded.block.cpp": "cpp",
"meta.embedded.block.diff": "diff",
"meta.embedded.block.dockerfile": "dockerfile",
"meta.embedded.block.go": "go",
"meta.embedded.block.groovy": "groovy",
"meta.embedded.block.pug": "jade",
"meta.embedded.block.javascript": "javascript",
"meta.embedded.block.javascriptreact": "javascriptreact",
"meta.embedded.block.json": "json",
"meta.embedded.block.less": "less",
"meta.embedded.block.objc": "objc",
"meta.embedded.block.scss": "scss",
"meta.embedded.block.perl6": "perl6",
"meta.embedded.block.powershell": "powershell",
"meta.embedded.block.python": "python",
"meta.embedded.block.rust": "rust",
"meta.embedded.block.scala": "scala",
"meta.embedded.block.shellscript": "shellscript",
"meta.embedded.block.typescript": "typescript",
"meta.embedded.block.typescriptreact": "typescriptreact",
"meta.embedded.block.csharp": "csharp",
"meta.embedded.block.fsharp": "fsharp"
"meta.embedded.c": "c",
"meta.embedded.cpp": "cpp",
"meta.embedded.cs": "csharp",
"meta.embedded.css": "css",
"meta.embedded.diff": "diff",
"meta.embedded.go": "go",
"meta.embedded.graphql": "graphql",
"meta.embedded.html": "html",
"meta.embedded.ini": "ini",
"meta.embedded.java": "java",
"meta.embedded.js": "javascriptreact",
"meta.embedded.json": "json",
"meta.embedded.kotlin": "kotlin",
"meta.embedded.less": "less",
"meta.embedded.lua": "lua",
"meta.embedded.makefile": "makefile",
"meta.embedded.md": "markdown",
"meta.embedded.mdx": "mdx",
"meta.embedded.objc": "objective-c",
"meta.embedded.perl": "perl",
"meta.embedded.php": "php",
"meta.embedded.python": "python",
"meta.embedded.r": "r",
"meta.embedded.ruby": "ruby",
"meta.embedded.rust": "rust",
"meta.embedded.scss": "scss",
"meta.embedded.shell": "shellscript",
"meta.embedded.sql": "sql",
"meta.embedded.svg": "xml",
"meta.embedded.swift": "swift",
"meta.embedded.toml": "toml",
"meta.embedded.ts": "typescriptreact",
"meta.embedded.tsx": "typescriptreact",
"meta.embedded.vbnet": "vb",
"meta.embedded.xml": "xml",
"meta.embedded.yaml": "yaml"
}
}
]
Expand Down
50 changes: 50 additions & 0 deletions packages/vscode-mdx/script/build.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#!/usr/bin/env node
import assert from 'node:assert/strict'
import fs from 'node:fs/promises'
import {fetch} from 'undici'

const repo = 'wooorm/markdown-tm-language'
const branch = 'main'
const filename = 'source.mdx.tmLanguage'

const branchesResponse = await fetch(
`https://api.github.com/repos/${repo}/branches`
)

const branches =
/** @type {Array<{name: string, commit: {sha: string, url: string}, protected: boolean}>} */ (
await branchesResponse.json()
)
const main = branches.find((d) => d.name === branch)
assert(main, 'expected `' + branch + '` branch')

const sha = main.commit.sha

const blobResponse = await fetch(
`https://raw.githubusercontent.com/${repo}/${branch}/${filename}`
)
let blob = await blobResponse.text()

let injected = false

blob = blob.replace(/<dict>/, ($0) => {
injected = true
return `<!--
This file is maintained at <https://github.com/${repo}/blob/${branch}/${filename}>.
To improve it, please create a pull request to the original repository.
Once accepted there, it can be pulled into this project (\`vscode-mdx\`) with
\`script/build.js\`, and released.
Version from SHA: ${sha}.
-->
${$0}`
})

assert(injected, 'expected to find a dict')

await fs.writeFile(
new URL('../syntaxes/source.mdx.tmLanguage', import.meta.url),
blob
)

console.log('Wrote `source.mdx.tmLanguage` at `' + sha + '`')
83 changes: 0 additions & 83 deletions packages/vscode-mdx/syntaxes/mdx.tmLanguage.json

This file was deleted.

Loading

0 comments on commit f1bd49b

Please sign in to comment.