Code Import

.gitignore

@@ -106,5 +106,12 @@ package-lock.json
 # translations are stored in the `i18n` via crowdin
 i18n
 
+
+# code-import
+code/node_modules  
+code/package-lock.json  
+code/yarn.lock  
+code/pnpm-lock.yaml  
+
 # vscode configuration
-.vscode
+.vscode

.husky/.gitignore

@@ -0,0 +1 @@
+_

.husky/pre-commit

@@ -0,0 +1 @@
+npx lint-staged

CONTRIBUTING.md

@@ -21,6 +21,9 @@ transparent as possible, whether it's:
 - publicly displayed via the UI of [solana.com](https://solana.com) (located in
   a different repo)
 - content translations are supported via Crowdin
+- code blocks must use code-import for file snippets (via filesystem)
+- code file should be [tests](https://nodejs.org/api/test.html) and should add
+  code ranges instead of whole test file
 
 ## Style guidelines
 
@@ -273,6 +276,52 @@ For images, you can use the path starting with `/public` like this:
 > links will be automatically adjusted to function on the website. Including
 > making the images viewable and removing `.md` file extensions.
 
+### Code Blocks
+
+In addition to standard markdown "fenced" code blocks (i.e. using triple
+backticks), the developer content repo requires the use of code-import for file
+snippets. This ensures that code examples are always up-to-date with the actual
+source files.
+
+#### Using code-import
+
+To use code-import, follow these steps:
+
+Ensure your code file is a test file located in the appropriate directory within
+the repo. Use the following syntax to import code snippets:
+
+```javascript file="/path/to/your/file.ts#L1-L10,#L15-L20"
+
+```
+
+This will import lines 1-10 and 15-20 from the specified file.
+
+Always use code ranges instead of importing whole files. This helps keep
+examples concise and focused.
+
+#### Code-import Rules
+
+- The file path must start with a forward slash (/).
+- You can specify multiple line ranges, separated by commas.
+- Line ranges should be in ascending order and not overlap.
+- Invalid ranges (e.g., #L4-L3) are not allowed.
+- Line numbers start at 1, so #L0 is invalid.
+- Trailing commas in the range specification are not allowed.
+
+Example of a valid code-import:
+
+```javascript file="/code/cookbook/wallets/check-public-key.ts#L1-L2,#L3-L18"
+
+```
+
+Example of an invalid code-import:
+
+```javascript file=/code/cookbook/wallets/check-public-key.ts#L1-L2,#L3-L19,#L1-L3
+
+```
+
+This is invalid because the ranges are not in ascending order and overlap.
+
 ### Table of contents
 
 When a content page is rendered on solana.com, a table of contents will be
@@ -519,7 +568,7 @@ a list of available components
   content
 - [images](#images) - details about how to include images in a piece of content
 - [code blocks](#code-blocks) - additional functionality on top of standard
-  markdown code blocks
+  markdown code blocks, these support code file import from filesystem
 - [blockquote](#blockquote) - additional functionality on top of the standard
   HTML `blockquote` element
 - [Callout](#callout) - custom component used to render message to the reader in

code/cookbook/wallets/check-public-key.ts

@@ -0,0 +1,19 @@
+import { PublicKey } from "@solana/web3.js";
+
+// Note that Keypair.generate() will always give a public key that is valid for users
+
+// Valid public key
+const key = new PublicKey("5oNDL3swdJJF1g9DzJiZ4ynHXgszjAEpUkxVYejchzrY");
+// Lies on the ed25519 curve and is suitable for users
+console.log(PublicKey.isOnCurve(key.toBytes()));
+
+// Valid public key
+const offCurveAddress = new PublicKey(
+  "4BJXYkfvg37zEmBbsacZjeQDpTNx91KppxFJxRqrz48e",
+);
+
+// Not on the ed25519 curve, therefore not suitable for users
+console.log(PublicKey.isOnCurve(offCurveAddress.toBytes()));
+
+// Not a valid public key
+const errorPubkey = new PublicKey("testPubkey");

code/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "code",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@solana/web3.js": "^1.95.2"
+  }
+}

coder.ts

@@ -0,0 +1,222 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+import remarkStringify from "remark-stringify";
+import remarkFrontmatter from "remark-frontmatter";
+import { visit } from "unist-util-visit";
+import ignore, { type Ignore } from "ignore";
+import importCode from "./src/utils/code-import";
+import chokidar from "chokidar";
+
+let debugMode = false;
+
+const debug = (...args: string[]) => {
+  if (debugMode) {
+    console.log("[DEBUG]", ...args);
+  }
+};
+
+const hasCodeComponentWithFileMeta = async (
+  filePath: string,
+): Promise<boolean> => {
+  const content = await fs.readFile(filePath, "utf8");
+  let hasMatch = false;
+
+  const tree = unified().use(remarkParse).use(remarkFrontmatter).parse(content);
+
+  visit(tree, "code", node => {
+    if (node.meta?.includes("file=")) {
+      hasMatch = true;
+      return false; // Stop visiting
+    }
+  });
+
+  return hasMatch;
+};
+
+const getIgnore = async (directory: string): Promise<Ignore> => {
+  const ig = ignore();
+
+  try {
+    const gitignoreContent = await fs.readFile(
+      path.join(directory, ".gitignore"),
+      "utf8",
+    );
+    ig.add(gitignoreContent);
+    // ignore all dotfiles
+    ig.add([".*"]);
+    // ignore CONTRIBUTING.md because it mentions the code component example
+    ig.add("CONTRIBUTING.md");
+  } catch (error) {
+    // If .gitignore doesn't exist, just continue without it
+    if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+      throw error;
+    }
+  }
+
+  return ig;
+};
+
+const getMarkdownAndMDXFiles = async (directory: string): Promise<string[]> => {
+  const ig = await getIgnore(directory);
+
+  const walkDir = async (dir: string): Promise<string[]> => {
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    const files = await Promise.all(
+      entries.map(async entry => {
+        const res = path.resolve(dir, entry.name);
+        const relativePath = path.relative(directory, res);
+
+        if (ig.ignores(relativePath) || entry.name === ".gitignore") {
+          debug(`Ignoring file: ${relativePath}`);
+          return [];
+        }
+
+        if (entry.isDirectory()) {
+          return walkDir(res);
+        }
+
+        if (
+          entry.isFile() &&
+          (entry.name.endsWith(".md") || entry.name.endsWith(".mdx"))
+        ) {
+          if (await hasCodeComponentWithFileMeta(res)) {
+            debug(`Found file with code component: ${relativePath}`);
+            return res;
+          }
+          debug(
+            `Skipping file (no code component with file meta): ${relativePath}`,
+          );
+        }
+
+        return [];
+      }),
+    );
+    return files.flat();
+  };
+
+  return walkDir(directory);
+};
+
+const processContent = async (
+  content: string,
+  filePath: string,
+): Promise<string> => {
+  try {
+    const file = await unified()
+      .use(remarkParse)
+      .use(remarkFrontmatter)
+      .use(importCode, {
+        preserveTrailingNewline: false,
+        removeRedundantIndentations: true,
+        rootDir: process.cwd(),
+      })
+      .use(remarkStringify, {
+        bullet: "-",
+        emphasis: "*",
+        fences: true,
+        listItemIndent: "one",
+        rule: "-",
+        ruleSpaces: false,
+        strong: "*",
+        tightDefinitions: true,
+      })
+      .process(content);
+    return String(file);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      throw new Error(
+        `File not found: ${(error as NodeJS.ErrnoException).path}`,
+      );
+    }
+    throw error;
+  }
+};
+
+const processFile = async (filePath: string): Promise<void> => {
+  try {
+    if (!(await hasCodeComponentWithFileMeta(filePath))) {
+      debug(`Skipping ${filePath}: No code component with file meta found.`);
+      return;
+    }
+
+    const originalContent = await fs.readFile(filePath, "utf8");
+    const processedContent = await processContent(originalContent, filePath);
+    if (originalContent !== processedContent) {
+      await fs.writeFile(filePath, processedContent);
+      console.log(`Updated: ${filePath}`);
+    } else {
+      debug(`No changes needed for: ${filePath}`);
+    }
+  } catch (error) {
+    console.error(`Error processing ${filePath}: ${(error as Error).message}`);
+  }
+};
+
+const processInChunks = async <T>(
+  items: T[],
+  processItem: (item: T) => Promise<void>,
+  chunkSize: number,
+): Promise<void> => {
+  for (let i = 0; i < items.length; i += chunkSize) {
+    const chunk = items.slice(i, i + chunkSize);
+    await Promise.all(chunk.map(processItem));
+  }
+};
+
+const watchFiles = async (directory: string): Promise<void> => {
+  const watcher = chokidar.watch(["**/*.md", "**/*.mdx"], {
+    ignored: [
+      "**.**",
+      /(^|[\/\\])\../,
+      "**/node_modules/**",
+      "**/.git/**",
+      ".gitignore",
+    ], // ignore dotfiles, node_modules, .git, and .gitignore
+    persistent: true,
+    cwd: directory,
+  });
+
+  console.log("Watch mode started. Waiting for file changes...");
+
+  watcher
+    .on("add", filePath => processFile(path.join(directory, filePath)))
+    .on("change", filePath => processFile(path.join(directory, filePath)))
+    .on("unlink", filePath => console.log(`File ${filePath} has been removed`));
+};
+
+const main = async (): Promise<void> => {
+  const filePath = process.argv[2];
+  const watchMode =
+    process.argv.includes("--watch") || process.argv.includes("-w");
+  debugMode = process.argv.includes("--debug") || process.argv.includes("-d");
+
+  if (debugMode) {
+    console.log("Debug mode enabled");
+  }
+
+  if (filePath && !watchMode && !debugMode) {
+    // Process single file
+    const absolutePath = path.resolve(process.cwd(), filePath);
+    console.log(`Processing single file: ${absolutePath}`);
+    await processFile(absolutePath);
+  } else if (watchMode) {
+    // Watch mode
+    await watchFiles(process.cwd());
+  } else {
+    // Process all files
+    const files = await getMarkdownAndMDXFiles(process.cwd());
+    const chunkSize = Math.max(1, Math.ceil(files.length / os.cpus().length));
+
+    console.log(`Processing ${files.length} files...`);
+    await processInChunks(files, processFile, chunkSize);
+  }
+
+  if (!watchMode) {
+    console.log("Sync process completed.");
+  }
+};
+
+main().catch(console.error);

content/cookbook/wallets/check-publickey.md

@@ -11,7 +11,7 @@ have a private key associated with them. You can check this by looking to see if
 the public key lies on the ed25519 curve. Only public keys that lie on the curve
 can be controlled by users with wallets.
 
-```javascript file="check-public-key.ts"
+```javascript file=/code/cookbook/wallets/check-public-key.ts#L1-L2,#L3-L19
 import { PublicKey } from "@solana/web3.js";
 
 // Note that Keypair.generate() will always give a public key that is valid for users