diff --git a/.github/workflows/feedbot.yml b/.github/workflows/feedbot.yml index 834d12d..784a153 100644 --- a/.github/workflows/feedbot.yml +++ b/.github/workflows/feedbot.yml @@ -7,29 +7,36 @@ jobs: rss-to-slack: runs-on: ubuntu-latest steps: + - name: Generate cache key + uses: actions/github-script@v6 + id: generate-key + with: + script: | + core.setOutput('cache-key', new Date().valueOf()) - name: Retrieve cache uses: actions/cache@v2 with: - path: ~/slackfeedbot - key: slackfeedbot-cache + path: ./slackfeedbot-cache + key: feed-cache-${{ steps.generate-key.outputs.cache-key }} + restore-keys: feed-cache- - name: NYT uses: 'selfagency/slackfeedbot@dev' with: rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} - cache_dir: ~/slackfeedbot + cache_dir: ./slackfeedbot-cache unfurl: false - name: LAT uses: 'selfagency/slackfeedbot@dev' with: rss: 'https://www.latimes.com/rss2.0.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} - cache_dir: ~/slackfeedbot + cache_dir: ./slackfeedbot-cache unfurl: false - name: WaPo uses: 'selfagency/slackfeedbot@dev' with: rss: 'https://feeds.washingtonpost.com/rss/homepage' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} - cache_dir: ~/slackfeedbot + cache_dir: ./slackfeedbot-cache unfurl: false \ No newline at end of file diff --git a/.husky/post-receive b/.husky/post-receive new file mode 100755 index 0000000..d4bed2b --- /dev/null +++ b/.husky/post-receive @@ -0,0 +1,4 @@ +#!/bin/sh +. "$(dirname "$0")/_/husky.sh" + +npm i diff --git a/README.md b/README.md index 84241cd..5bcd875 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Push RSS feed updates to Slack via GitHub Actions ### With cache folder +Hashes and caches the post title + creation date to ensure no duplicates are posted. + ``` name: SlackFeedBot on: @@ -39,13 +41,19 @@ jobs: rss-to-slack: runs-on: ubuntu-latest steps: + - name: Generate cache key + uses: actions/github-script@v6 + id: generate-key + with: + script: | + core.setOutput('cache-key', new Date().valueOf()) - name: Retrieve cache uses: actions/cache@v2 with: - path: ~/slackfeedbot - key: slackfeedbot-cache + path: ./slackfeedbot-cache + key: feed-cache-${{ steps.generate-key.outputs.cache-key }} - name: NYT - uses: 'selfagency/feedbot@v1.2.2' + uses: 'selfagency/feedbot@v1.2.3' with: rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} @@ -54,6 +62,8 @@ jobs: ### With interval +No cache, but maybe duplicates. + ``` name: SlackFeedBot on: @@ -64,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: NYT - uses: 'selfagency/feedbot@v1.2.2' + uses: 'selfagency/feedbot@v1.2.3' with: rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} @@ -82,13 +92,19 @@ jobs: rss-to-slack: runs-on: ubuntu-latest steps: + - name: Generate cache key + uses: actions/github-script@v6 + id: generate-key + with: + script: | + core.setOutput('cache-key', new Date().valueOf()) - name: Retrieve cache uses: actions/cache@v2 with: - path: ~/slackfeedbot - key: slackfeedbot-cache + path: ./slackfeedbot-cache + key: feed-cache-${{ steps.generate-key.outputs.cache-key }} - name: NYT - uses: 'selfagency/feedbot@v1.2.2' + uses: 'selfagency/feedbot@v1.2.3' with: rss: 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} @@ -107,19 +123,25 @@ jobs: rss-to-slack: runs-on: ubuntu-latest steps: + - name: Generate cache key + uses: actions/github-script@v6 + id: generate-key + with: + script: | + core.setOutput('cache-key', new Date().valueOf()) - name: Retrieve cache uses: actions/cache@v2 with: - path: ~/slackfeedbot - key: slackfeedbot-cache + path: ./slackfeedbot-cache + key: feed-cache-${{ steps.generate-key.outputs.cache-key }} - name: LAT - uses: 'selfagency/feedbot@v1.2.2' + uses: 'selfagency/feedbot@v1.2.3' with: rss: 'https://www.latimes.com/rss2.0.xml' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} cache_dir: '~/slackfeedbot' - name: WaPo - uses: 'selfagency/feedbot@v1.2.2' + uses: 'selfagency/feedbot@v1.2.3' with: rss: 'https://feeds.washingtonpost.com/rss/homepage' slack_webhook: ${{ secrets.SLACK_WEBHOOK }} diff --git a/action.js b/action.js index 5b86726..6dd4bf9 100644 --- a/action.js +++ b/action.js @@ -1,16 +1,18 @@ +import { mkdir, readFile, writeFile } from 'fs'; + +import { compile } from 'html-to-text'; import core from '@actions/core'; import { createHash } from 'crypto'; import dayjs from 'dayjs'; -import { readFile, writeFile } from 'fs'; -import html2md from 'html-to-md'; -import { compile } from 'html-to-text'; import fetch from 'node-fetch'; +import html2md from 'html-to-md'; import { parse } from 'rss-to-json'; import { promisify } from 'util'; const read = promisify(readFile); const write = promisify(writeFile); -const { debug, setFailed, getInput } = core; +const md = promisify(mkdir); +const { debug, setFailed, getInput, getBooleanInput } = core; const html2txt = compile({ wordwrap: 120 }); @@ -70,7 +72,7 @@ const run = async () => { const rssFeedUrl = new URL(rssFeed); const slackWebhook = getInput('slack_webhook'); const interval = parseInt(getInput('interval')); - const unfurl = getInput('unfurl').toString() === 'true'; + const unfurl = getBooleanInput('unfurl'); const cacheDir = getInput('cache_dir'); const cachePath = `${cacheDir}/${rssFeedUrl.hostname.replace(/\./g, '_')}.json`; @@ -80,17 +82,16 @@ const run = async () => { debug('Checking for feed items'); if (rss?.items?.length) { - debug(`Selecting items posted in the last ${interval} minutes`); - let toSend = []; let published = []; if (cacheDir) { debug(`Retrieving previously published entries`); try { - published = JSON.stringify(await read(cachePath, 'utf8')); + published = JSON.parse(await read(cachePath, 'utf8')); + debug(published); toSend = rss.items.filter(item => { - return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.description))); + return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.created))); }); } catch (err) { debug(err.message); @@ -99,6 +100,7 @@ const run = async () => { }); } } else { + debug(`Selecting items posted in the last ${interval} minutes`); toSend = rss.items.filter(item => { return dayjs(item.created).isAfter(dayjs().subtract(interval, 'minute')); }); @@ -117,7 +119,7 @@ const run = async () => { } if (item.link) text += `<${item.link}|Read more>`; } else { - if (item.title) text += `<${item.link}|${html2txt(item.title)}>`; + if (item.title) text += `<${item.link}|${html2txt(item.title + item.created)}>`; } return { @@ -154,9 +156,15 @@ const run = async () => { if (cacheDir) { debug(`Writing cache to ${cachePath}`); + try { + await md(cacheDir, { recursive: true }); + } catch (err) { + debug(err.message); + } + await write( cachePath, - JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title + item.description)))]) + JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title)))]) ); } } diff --git a/dist/index.js b/dist/index.js index 18e548c..501234d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -21347,18 +21347,16 @@ var __webpack_exports__ = {}; // This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk. (() => { +// EXTERNAL MODULE: external "fs" +var external_fs_ = __nccwpck_require__(7147); +// EXTERNAL MODULE: ./node_modules/html-to-text/index.js +var html_to_text = __nccwpck_require__(7015); // EXTERNAL MODULE: ./node_modules/@actions/core/lib/core.js var core = __nccwpck_require__(2186); ;// CONCATENATED MODULE: external "crypto" const external_crypto_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("crypto"); // EXTERNAL MODULE: ./node_modules/dayjs/dayjs.min.js var dayjs_min = __nccwpck_require__(7401); -// EXTERNAL MODULE: external "fs" -var external_fs_ = __nccwpck_require__(7147); -// EXTERNAL MODULE: ./node_modules/html-to-md/dist/index.js -var dist = __nccwpck_require__(7192); -// EXTERNAL MODULE: ./node_modules/html-to-text/index.js -var html_to_text = __nccwpck_require__(7015); ;// CONCATENATED MODULE: external "node:http" const external_node_http_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:http"); ;// CONCATENATED MODULE: external "node:https" @@ -21421,7 +21419,7 @@ function dataUriToBuffer(uri) { buffer.charset = charset; return buffer; } -/* harmony default export */ const data_uri_to_buffer_dist = (dataUriToBuffer); +/* harmony default export */ const dist = (dataUriToBuffer); //# sourceMappingURL=index.js.map ;// CONCATENATED MODULE: external "node:util" const external_node_util_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.url)("node:util"); @@ -23112,7 +23110,7 @@ async function fetch(url, options_) { } if (parsedURL.protocol === 'data:') { - const data = data_uri_to_buffer_dist(request.url); + const data = dist(request.url); const response = new Response(data, {headers: {'Content-Type': data.typeFull}}); resolve(response); return; @@ -23466,6 +23464,8 @@ function fixResponseChunkedTransferBadEnding(request, errorCallback) { }); } +// EXTERNAL MODULE: ./node_modules/html-to-md/dist/index.js +var html_to_md_dist = __nccwpck_require__(7192); // EXTERNAL MODULE: ./node_modules/rss-to-json/dist/index.js var rss_to_json_dist = __nccwpck_require__(7235); // EXTERNAL MODULE: external "util" @@ -23481,9 +23481,11 @@ var external_util_ = __nccwpck_require__(3837); + const read = (0,external_util_.promisify)(external_fs_.readFile); const write = (0,external_util_.promisify)(external_fs_.writeFile); -const { debug, setFailed, getInput } = core; +const md = (0,external_util_.promisify)(external_fs_.mkdir); +const { debug, setFailed, getInput, getBooleanInput } = core; const html2txt = (0,html_to_text.compile)({ wordwrap: 120 }); @@ -23543,7 +23545,7 @@ const run = async () => { const rssFeedUrl = new URL(rssFeed); const slackWebhook = getInput('slack_webhook'); const interval = parseInt(getInput('interval')); - const unfurl = getInput('unfurl').toString() === 'true'; + const unfurl = getBooleanInput('unfurl'); const cacheDir = getInput('cache_dir'); const cachePath = `${cacheDir}/${rssFeedUrl.hostname.replace(/\./g, '_')}.json`; @@ -23553,17 +23555,16 @@ const run = async () => { debug('Checking for feed items'); if (rss?.items?.length) { - debug(`Selecting items posted in the last ${interval} minutes`); - let toSend = []; let published = []; if (cacheDir) { debug(`Retrieving previously published entries`); try { - published = JSON.stringify(await read(cachePath, 'utf8')); + published = JSON.parse(await read(cachePath, 'utf8')); + debug(published); toSend = rss.items.filter(item => { - return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.description))); + return !published.find(pubbed => pubbed === hash(JSON.stringify(item.title + item.created))); }); } catch (err) { debug(err.message); @@ -23572,6 +23573,7 @@ const run = async () => { }); } } else { + debug(`Selecting items posted in the last ${interval} minutes`); toSend = rss.items.filter(item => { return dayjs_min(item.created).isAfter(dayjs_min().subtract(interval, 'minute')); }); @@ -23583,14 +23585,14 @@ const run = async () => { if (!unfurl) { if (item.title) text += `*${html2txt(item.title)}*\n`; if (item.description) { - const description = dist(item.description) + const description = html_to_md_dist(item.description) .replace(/[Rr]ead more/g, '…') .replace(/\n/g, ' '); text += `${description}\n`; } if (item.link) text += `<${item.link}|Read more>`; } else { - if (item.title) text += `<${item.link}|${html2txt(item.title)}>`; + if (item.title) text += `<${item.link}|${html2txt(item.title + item.created)}>`; } return { @@ -23627,9 +23629,15 @@ const run = async () => { if (cacheDir) { debug(`Writing cache to ${cachePath}`); + try { + await md(cacheDir, { recursive: true }); + } catch (err) { + debug(err.message); + } + await write( cachePath, - JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title + item.description)))]) + JSON.stringify([...published, ...toSend.map(item => hash(JSON.stringify(item.title)))]) ); } } diff --git a/package-lock.json b/package-lock.json index 65f37ae..8a35ece 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "slackfeedbot", - "version": "1.2.2", + "version": "1.2.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "slackfeedbot", - "version": "1.2.2", + "version": "1.2.3", "license": "MIT", "devDependencies": { "@actions/core": "^1.6.0", diff --git a/package.json b/package.json index deb3e32..38d1df9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "slackfeedbot", - "version": "1.2.2", + "version": "1.2.3", "license": "MIT", "author": "Daniel Sieradski ", "description": "Push RSS feed updates to Slack via GitHub Actions",