ci: add link check workflow

.github/workflows/check_links.yml

@@ -0,0 +1,27 @@
+name: Check Links
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-links:
+    name: Check Links
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+      - name: Install dependencies
+      - run: yarn install --frozen-lockfile
+      - name: Check Links
+        run: yarn run check-links

package.json

@@ -2,7 +2,8 @@
   "scripts": {
     "test": "jest",
     "generate_rule": "node scripts/generate_rule.js",
-    "bulk-update": "node scripts/bulk_update.js"
+    "bulk-update": "node scripts/bulk_update.js",
+    "check-links": "node scripts/check_links.js"
   },
   "devDependencies": {
     "jest": "^29.6.1",
@@ -11,6 +12,8 @@
   "dependencies": {
     "commander": "^11.1.0",
     "csv-parse": "^5.5.3",
+    "glob": "^10.3.10",
+    "marked": "^12.0.1",
     "yaml": "^2.3.4"
   }
 }

scripts/check_links.js

@@ -0,0 +1,93 @@
+const path = require('path')
+const fs = require('fs')
+const http = require('http')
+const https = require('https')
+
+const YAML = require('yaml')
+const glob = require('glob')
+const { marked } = require('marked')
+
+const rules = glob.globSync('**/*.{yml,yaml}', { root: path.resolve(__dirname, '../rules') })
+
+const headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0' }
+
+const validURLs = new Map()
+const promises = []
+const failures = []
+let failed = false
+
+for (const filename of rules) {
+  const rule = YAML.parse(fs.readFileSync(filename, 'utf8'))
+  const remediationMessage = rule.metadata.remediation_message || ''
+
+  const urls = []
+
+  const visitAll = (tokens) => {
+    if (!tokens) {
+      return
+    }
+
+    for (const token of tokens) {
+      visit(token)
+    }
+  }
+
+  const visit = (token) => {
+    if (token.type == "link") {
+      if (token.href) {
+        urls.push(token.href)
+      }
+    }
+
+    visitAll(token.tokens)
+    visitAll(token.items)
+  }
+
+  visitAll(marked.lexer(remediationMessage))
+
+  for (const url of urls) {
+    if (validURLs.has(url)) {
+      if (!validURLs.get(url)) {
+        failures.push({ filename, url })
+      }
+
+      continue
+    }
+
+    const protocol = url.startsWith('https') ? https : http
+
+    promises.push(new Promise((resolve) => {
+      protocol.request(url, { method: 'HEAD', headers }, (response) => {
+        if (response.statusCode == 200) {
+          console.info("PASS " + url)
+          validURLs.set(url, true)
+        } else {
+          console.error("FAIL " + url)
+          failures.push({ filename, url, status: response.statusCode })
+          failed = true
+          validURLs.set(url, false)
+        }
+
+        resolve()
+      }).on('error', (e) => {
+        console.error("FAIL " + url)
+        failures.push({ filename, url, message: e.message })
+        failed = true
+        validURLs.set(url, false)
+
+        resolve()
+      }).end()
+    }))
+  }
+}
+
+Promise.all(promises).then(() => {
+  if (failures.length != 0) {
+    console.log('\n\nFailures:\n\n')
+    console.log(failures)
+  }
+
+  if (failed) {
+    process.exit(1)
+  }
+})