From c5fc135afedc10058159af544a347c8a6514ce74 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 30 Mar 2024 22:38:37 +0000 Subject: [PATCH 1/4] feat: add Twitch support Closes #328 --- public/README.md | 8 ++++++++ src/providers/index.js | 1 + src/providers/twitch.js | 20 ++++++++++++++++++++ 3 files changed, 29 insertions(+) create mode 100644 src/providers/twitch.js diff --git a/public/README.md b/public/README.md index d7a554a..85b8a33 100644 --- a/public/README.md +++ b/public/README.md @@ -202,6 +202,14 @@ It resolves user avatar against **twitter.com**. e.g., https://unavatar.io/twitter/kikobeats +### Twitch + +Type: `username` + +It resolves user avatar against **twitch.tv**. + +e.g., https://unavatar.io/twitter/midudev + ### YouTube Type: `username` diff --git a/src/providers/index.js b/src/providers/index.js index b5c18fd..2d4f406 100644 --- a/src/providers/index.js +++ b/src/providers/index.js @@ -18,6 +18,7 @@ const providers = { substack: require('./substack'), telegram: require('./telegram'), tiktok: require('./tiktok'), + twitch: require('./twitch'), twitter: require('./twitter'), youtube: require('./youtube') } diff --git a/src/providers/twitch.js b/src/providers/twitch.js new file mode 100644 index 0000000..ff573f6 --- /dev/null +++ b/src/providers/twitch.js @@ -0,0 +1,20 @@ +'use strict' + +const PCancelable = require('p-cancelable') +const cheerio = require('cheerio') + +const getHTML = require('../util/html-get') + +module.exports = PCancelable.fn(async function instagram ({ input }, onCancel) { + const promise = getHTML(`https://www.twitch.tv/${input}`) + onCancel(() => promise.onCancel()) + const { html } = await promise + const $ = cheerio.load(html) + return $('meta[property="og:image"]').attr('content') +}) + +module.exports.supported = { + email: false, + username: true, + domain: false +} From 84393bd7d2b29e5a1aabf660883ab6f103df8669 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 30 Mar 2024 23:04:07 +0000 Subject: [PATCH 2/4] fix: third party integrations --- package.json | 9 ++++----- src/providers/deviantart.js | 20 +++++--------------- src/providers/gitlab.js | 11 +---------- src/providers/soundcloud.js | 10 +++++----- test/providers.js | 12 ++++++++++-- 5 files changed, 25 insertions(+), 37 deletions(-) diff --git a/package.json b/package.json index 0670f88..af5cabc 100644 --- a/package.json +++ b/package.json @@ -93,7 +93,7 @@ "@keyvhq/redis": "~2.1.0", "@kikobeats/time-span": "~1.0.4", "@microlink/mql": "~0.13.4", - "@microlink/ping-url": "~1.4.12", + "@microlink/ping-url": "~1.4.14", "@microlink/ua": "~1.2.3", "async-memoize-one": "~1.1.7", "browserless": "~10.4.0", @@ -107,7 +107,7 @@ "frequency-counter": "~1.0.1", "got": "~11.8.6", "helmet": "~7.1.0", - "html-get": "~2.16.1", + "html-get": "~2.16.3", "http-compression": "~1.0.19", "https-tls": "~1.0.16", "ioredis": "~5.3.2", @@ -121,8 +121,7 @@ "p-cancelable": "2.1.1", "p-reflect": "~2.1.0", "p-timeout": "~4.1.0", - "puppeteer": "~22.6.0", - "qsm": "~2.1.2", + "puppeteer": "~22.6.1", "rate-limiter-flexible": "~5.0.0", "router-http": "~1.0.7", "send-http": "~1.0.6", @@ -130,7 +129,7 @@ "srcset": "~4.0.0", "tangerine": "~1.5.4", "top-crawler-agents": "~1.0.26", - "top-user-agents": "~2.1.12", + "top-user-agents": "~2.1.17", "unique-random-array": "~2.0.0", "url-regex": "~5.0.0" }, diff --git a/src/providers/deviantart.js b/src/providers/deviantart.js index c1d390b..f2f17c0 100644 --- a/src/providers/deviantart.js +++ b/src/providers/deviantart.js @@ -5,25 +5,15 @@ const cheerio = require('cheerio') const getHTML = require('../util/html-get') -const REGEX_PROFILE_URL = - /^https?:\/\/(?:www\.)?deviantart\.com\/([\w-]+)(?:\/.+)?$/ - -module.exports = PCancelable.fn(async function deviantart ({ input }, onCancel) { +module.exports = PCancelable.fn(async function deviantart ( + { input }, + onCancel +) { const promise = getHTML(`https://www.deviantart.com/${input}`) onCancel(() => promise.onCancel()) const { html } = await promise const $ = cheerio.load(html) - const canonUsername = $('head link[rel=canonical]') - .attr('href') - .replace(REGEX_PROFILE_URL, '$1') - const els = $('img.avatar,a[data-hook=user_link][data-icon]').filter( - (i, el) => { - const thisUsername = - $(el).attr('data-username') || $(el).attr('title') || '' - return canonUsername.toLowerCase() === thisUsername.toLowerCase() - } - ) - return els.attr('data-icon') || els.attr('src') + return $('meta[property="og:image"]').attr('content') }) module.exports.supported = { diff --git a/src/providers/gitlab.js b/src/providers/gitlab.js index a885833..ebbbc60 100644 --- a/src/providers/gitlab.js +++ b/src/providers/gitlab.js @@ -2,24 +2,15 @@ const PCancelable = require('p-cancelable') const cheerio = require('cheerio') -const qsm = require('qsm') const getHTML = require('../util/html-get') -const { AVATAR_SIZE } = require('../constant') - module.exports = PCancelable.fn(async function gitlab ({ input }, onCancel) { const promise = getHTML(`https://gitlab.com/${input}`) onCancel(() => promise.onCancel()) const { html } = await promise const $ = cheerio.load(html) - - let avatarUrl = $('.avatar-holder > a > img').attr('src') - if (avatarUrl) avatarUrl = `https://gitlab.com${avatarUrl}` - - return qsm.exist(avatarUrl, 'width') - ? qsm.add(avatarUrl, { width: AVATAR_SIZE }) - : avatarUrl + return $('meta[property="og:image"]').attr('content') }) module.exports.supported = { diff --git a/src/providers/soundcloud.js b/src/providers/soundcloud.js index 4759eb3..e93cfce 100644 --- a/src/providers/soundcloud.js +++ b/src/providers/soundcloud.js @@ -5,15 +5,15 @@ const cheerio = require('cheerio') const getHTML = require('../util/html-get') -module.exports = PCancelable.fn(async function soundcloud ({ input }, onCancel) { +module.exports = PCancelable.fn(async function soundcloud ( + { input }, + onCancel +) { const promise = getHTML(`https://soundcloud.com/${input}`) onCancel(() => promise.onCancel()) const { html } = await promise const $ = cheerio.load(html, { xmlMode: true }) - const name = $(`a[itemprop=url][href="/${input}" i]`) - return $(`img[itemprop=image][alt="${name.text().trim()}’s avatar" i]`).attr( - 'src' - ) + return $('meta[property="og:image"]').attr('content') }) module.exports.supported = { diff --git a/test/providers.js b/test/providers.js index a06bd80..c6392f6 100644 --- a/test/providers.js +++ b/test/providers.js @@ -47,7 +47,7 @@ test('soundcloud', async t => { t.true(body.url.includes('images.weserv.nl')) }) // -;(isCI ? test.skip : test)('deviantart', async t => { +test('deviantart', async t => { const serverUrl = await runServer(t) const { body } = await got('deviantart/spyed?json', { prefixUrl: serverUrl @@ -119,6 +119,14 @@ test('substack', async t => { t.true(body.url.includes('images.weserv.nl')) }) +test('twitch', async t => { + const serverUrl = await runServer(t) + const { body } = await got('twitch/midudev?json', { + prefixUrl: serverUrl + }) + t.true(body.url.includes('images.weserv.nl')) +}) + test('microlink', async t => { const serverUrl = await runServer(t) const { body } = await got('microlink/teslahunt.io?json', { @@ -135,7 +143,7 @@ test('readcv', async t => { t.true(body.url.includes('images.weserv.nl')) }) // -;(isCI ? test.skip : test)('tiktok', async t => { +test.skip('tiktok', async t => { const serverUrl = await runServer(t) const { body } = await got('tiktok/carlosazaustre?json', { prefixUrl: serverUrl From f3ceb9c427cb9218a8e10fafb600627a55c60cb7 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 30 Mar 2024 23:06:46 +0000 Subject: [PATCH 3/4] chore: disable tiktok MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I didn't find a way to get the avatar ☹️ --- src/providers/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/index.js b/src/providers/index.js index 2d4f406..42b2273 100644 --- a/src/providers/index.js +++ b/src/providers/index.js @@ -17,7 +17,7 @@ const providers = { soundcloud: require('./soundcloud'), substack: require('./substack'), telegram: require('./telegram'), - tiktok: require('./tiktok'), + // tiktok: require('./tiktok'), twitch: require('./twitch'), twitter: require('./twitter'), youtube: require('./youtube') From 267964f9c750cff97e12e1f5f28cb71b39ede629 Mon Sep 17 00:00:00 2001 From: Kiko Beats Date: Sat, 30 Mar 2024 23:09:09 +0000 Subject: [PATCH 4/4] ci: skip --- test/providers.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/providers.js b/test/providers.js index c6392f6..17a69ba 100644 --- a/test/providers.js +++ b/test/providers.js @@ -94,8 +94,7 @@ test('telegram', async t => { }) t.true(body.url.includes('images.weserv.nl')) }) - -test('substack', async t => { +;(isCI ? test.skip : test)('substack', async t => { const serverUrl = await runServer(t) const { body } = await got('substack/bankless?json', { prefixUrl: serverUrl