Skip to content

Commit

Permalink
Merge pull request #237 from TorstenDittmann/229-feature-request-cust…
Browse files Browse the repository at this point in the history
…om-fencecodeblock-highlighting-function-in-preprocessor-config

feat: add highlighter function
TorstenDittmann authored Oct 13, 2024
2 parents 28f6056 + e60c44a commit 5737e0c
Showing 17 changed files with 369 additions and 118 deletions.
1 change: 0 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -18,5 +18,4 @@ jobs:
- run: corepack enable
- run: pnpm install --frozen-lockfile
- run: pnpm run build
- run: pnpx playwright install --with-deps chromium
- run: pnpm test
2 changes: 1 addition & 1 deletion apps/demo/package.json
Original file line number Diff line number Diff line change
@@ -6,7 +6,7 @@
"dev": "vite dev",
"build": "vite build",
"preview": "vite preview",
"test": "playwright test",
"test": "playwright install --with-deps && playwright test",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
2 changes: 1 addition & 1 deletion packages/process/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "svelte-markdoc-preprocess",
"version": "2.0.2",
"version": "2.1.0",
"description": "A Svelte preprocessor that allows you to use Markdoc.",
"type": "commonjs",
"keywords": [
4 changes: 4 additions & 0 deletions packages/process/src/config.ts
Original file line number Diff line number Diff line change
@@ -40,4 +40,8 @@ export type Config = {
* Configuration for the markdoc compiler.
*/
config: ConfigType | null;
/**
* A function that will provide a custom highlighter for code blocks.
*/
highlighter: ((code: string, language: string) => Promise<string>) | null;
};
5 changes: 4 additions & 1 deletion packages/process/src/processor.ts
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ const default_config: Config = {
config: null,
validationThreshold: 'error',
allowComments: false,
highlighter: null,
};

const processor = ({
@@ -24,6 +25,7 @@ const processor = ({
config = default_config.config,
validationThreshold = default_config.validationThreshold,
allowComments = default_config.allowComments,
highlighter: highlighter = default_config.highlighter,
}: Partial<Config> = default_config): PreprocessorGroup => {
return {
name: 'svelte-markdoc-preprocess',
@@ -40,7 +42,7 @@ const processor = ({
/**
* Add svelte components to be used with markdoc tags
*/
const code = transformer({
const code = await transformer({
filename,
config,
content,
@@ -51,6 +53,7 @@ const processor = ({
partials_dir: partials,
validation_threshold: validationThreshold,
allow_comments: allowComments,
highlighter,
});

return {
62 changes: 54 additions & 8 deletions packages/process/src/renderer.ts
Original file line number Diff line number Diff line change
@@ -3,23 +3,40 @@ import { sanitize_for_svelte } from './transformer';
import { escape } from 'html-escaper';
import { IMAGE_PREFIX, IMPORT_PREFIX, NODES_IMPORT } from './constants';
import { is_relative_path } from './utils';
import { Config } from './config';

export function render_html(
export async function render_html(
node: RenderableTreeNodes,
dependencies: Map<string, string>,
): string {
highlighter: Config['highlighter'],
escape_html = true,
): Promise<string> {
/**
* if the node is a string or number, it's a text node.
*/
if (typeof node === 'string' || typeof node === 'number') {
return sanitize_for_svelte(escape(String(node)));
if (escape_html) {
return sanitize_for_svelte(escape(String(node)));
} else {
return sanitize_for_svelte(String(node));
}
}

/**
* if the node is an array, render its items.
*/
if (Array.isArray(node)) {
return node.map((item) => render_html(item, dependencies)).join('');
return Promise.all(
node.map(
async (item) =>
await render_html(
item,
dependencies,
highlighter,
escape_html,
),
),
).then((items) => items.join(''));
}

/**
@@ -29,10 +46,10 @@ export function render_html(
return '';
}

const { name, attributes, children = [] } = node;
let { name, attributes, children = [] } = node;

if (!name) {
return render_html(children, dependencies);
return await render_html(children, dependencies, highlighter);
}

const is_svelte = is_svelte_component(node);
@@ -41,9 +58,10 @@ export function render_html(
* add attributes to the tag.
*/
let output = `<${name}`;
for (const [key, value] of Object.entries(attributes ?? {})) {
for (let [key, value] of Object.entries(attributes ?? {})) {
const is_src_key = key === 'src';
const is_imported_image = is_src_key && is_relative_path(value);

if (is_svelte) {
switch (name.toLowerCase()) {
case `${NODES_IMPORT}.image`.toLowerCase():
@@ -97,11 +115,39 @@ export function render_html(
return output;
}

let escape_next = true;

if (highlighter) {
const run_highlighter =
name.toLowerCase() === `${NODES_IMPORT}.fence`.toLowerCase() ||
name.toLowerCase() === 'pre'.toLowerCase();
if (run_highlighter) {
escape_next = false;
children = await Promise.all(
children.map(async (child) =>
typeof child === 'string'
? await highlighter(
child,
(is_svelte
? attributes?.language
: attributes['data-language']) ?? '',
)
: child,
),
);
}
}

/**
* render the children if present.
*/
if (children.length) {
output += render_html(children, dependencies);
output += await render_html(
children,
dependencies,
highlighter,
escape_next,
);
}

/**
16 changes: 7 additions & 9 deletions packages/process/src/transformer.ts
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ type Var = {
type: StringConstructor | NumberConstructor | BooleanConstructor;
};

export function transformer({
export async function transformer({
content,
filename,
nodes_file,
@@ -50,6 +50,7 @@ export function transformer({
config,
validation_threshold,
allow_comments,
highlighter,
}: {
content: string;
filename: string;
@@ -61,7 +62,8 @@ export function transformer({
config: Config['config'];
validation_threshold: Config['validationThreshold'];
allow_comments: Config['allowComments'];
}): string {
highlighter: Config['highlighter'];
}): Promise<string> {
/**
* create tokenizer
*/
@@ -189,7 +191,7 @@ export function transformer({
/**
* render to html
*/
const code = render_html(nast, dependencies);
const code = await render_html(nast, dependencies, highlighter);

let transformed = '';

@@ -406,14 +408,9 @@ function prepare_nodes(
if (nodes_file) {
for (const [name] of each_exported_var(nodes_file)) {
const type = name.toLowerCase() as NodeType;
if (type === 'image') {
}
nodes[name.toLowerCase()] = {
...get_node_defaults(type),
transform(node, config) {
if (type === 'image') {
node.attributes.src;
}
return new Tag(
`${NODES_IMPORT}.${name}`,
node.transformAttributes(config),
@@ -470,7 +467,8 @@ function each_exported_var(filepath: string): Array<[string, string]> {
if (node.type === 'ExportSpecifier') {
if (
parent?.type === 'ExportNamedDeclaration' &&
parent?.source
parent?.source &&
node.exported.type === 'Identifier'
) {
tup.push([node.exported.name, String(parent.source.value)]);
}
1 change: 1 addition & 0 deletions packages/process/tests/nodes/module.svelte
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
<script context="module">
export { default as Heading } from './mock.svelte';
export { default as Image } from './mock.svelte';
export { default as Fence } from './mock.svelte';
</script>
5 changes: 4 additions & 1 deletion packages/process/tests/processor.test.mjs
Original file line number Diff line number Diff line change
@@ -69,7 +69,10 @@ test('preprocessor', async (context) => {
files.map(async (entry) => {
return context.test('tests ' + basename(entry), async () => {
const before = read_file(join(entry, 'source.markdoc'));
const after = read_file(join(entry, 'compiled.txt'));
const after = read_file(join(entry, 'compiled.txt')).replace(
/\\n/g,
'\n',
);
const config = await import('../' + join(entry, 'config.mjs'));
const preprocess = config.default;
const exception = config.exception ?? false;
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<script>import * as INTERNAL__NODES from 'tests/nodes/module.svelte';</script><article><INTERNAL__NODES.Fence content="console.log(&#39;hello world&#39;);\n" language="javascript" process={true}>javascript:console.log('hello world');\n</INTERNAL__NODES.Fence><INTERNAL__NODES.Fence content="&lt;b&gt;bold&lt;/b&gt;\n" language="html" process={true}>html:<b>bold</b>\n</INTERNAL__NODES.Fence></article>
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { markdoc } from '../../../dist/module.js';
import { absoulute } from '../../utils.mjs';

export default markdoc({
nodes: absoulute(import.meta.url, '../../nodes/module.svelte'),
highlighter: (code, lang) => `${lang}:${code}`,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
```javascript
console.log('hello world');
```

```html
<b>bold</b>
```
1 change: 1 addition & 0 deletions packages/process/tests/processor/highlighter/compiled.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<article><pre data-language="javascript">javascript:console.log('hello world');\n</pre><pre data-language="html">html:<b>bold</b>\n</pre></article>
5 changes: 5 additions & 0 deletions packages/process/tests/processor/highlighter/config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import { markdoc } from '../../../dist/module.js';

export default markdoc({
highlighter: (code, lang) => `${lang}:${code}`,
});
7 changes: 7 additions & 0 deletions packages/process/tests/processor/highlighter/source.markdoc
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
```javascript
console.log('hello world');
```

```html
<b>bold</b>
```
359 changes: 264 additions & 95 deletions pnpm-lock.yaml

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pnpm-workspace.yaml
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ packages:

catalog:
'@sveltejs/kit': ^2.5.24
'@typescript-eslint/parser': ^7.0.0
'@typescript-eslint/parser': ^8.0.0
'prettier': ^3.1.1
'svelte': ^4.0.0
'typescript': ^5.5.2

0 comments on commit 5737e0c

Please sign in to comment.