fix: c/c++ directive and preprocessor tokenize (#26)

blurfx · Nov 25, 2023 · 65b99bf · 65b99bf
1 parent 5f3772b
commit 65b99bf
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/.changeset/rare-bees-talk.md b/.changeset/rare-bees-talk.md
@@ -0,0 +1,6 @@
+---
+'@calor/core': patch
+'@calor/highlighter': patch
+---
+
+Fix C/C++ directive and preprocessor tokenize rule"
diff --git a/packages/core/src/rules/cpp.ts b/packages/core/src/rules/cpp.ts
@@ -5,6 +5,15 @@ const cppRules: ParseRule[] = [
     kind: 'comment',
     pattern: /(\/\/.*\n?)|\/\*[^]*?\*\//g,
   },
+  {
+    kind: 'keyword',
+    pattern: /(#\s*include|ifdef|ifndef|elidef|elifndef)(\s*)(<.*>|".*")/g,
+    matchHints: ['keyword', 'plain', 'string'],
+  },
+  {
+    kind: 'keyword',
+    pattern: /(#\s*(?:define|if|elif|else|endif|import|error|line))\b/g,
+  },
   {
     kind: 'keyword',
     pattern:

diff --git a/packages/core/tests/tokenizer/c-cpp.test.ts b/packages/core/tests/tokenizer/c-cpp.test.ts
@@ -112,10 +112,8 @@ describe('c/cpp tokenizer', () => {
   it('can tokenize string', () => {
     const tokens = tokenize(
       `
-      #include <iostream>
       int main() {
         std::cout << "Hello, World!";
-        return 0;
       }
     `,
       cppRules,
@@ -124,4 +122,11 @@ describe('c/cpp tokenizer', () => {
     expect(stringTokens.length).toBe(1);
     expect(stringTokens[0].value).toBe('"Hello, World!"');
   });
+
+  it('include directive', () => {
+    const tokens = tokenize('#include <iostream>', cppRules);
+    const includeTokens = tokens.filter((token) => token.kind === 'keyword');
+    expect(includeTokens.length).toBe(1);
+    expect(includeTokens[0].value).toBe('#include');
+  });
 });