diff --git a/package.json b/package.json index babce07..befeff8 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "@keyvhq/multi": "~2.1.0", "@keyvhq/redis": "~2.1.0", "@kikobeats/time-span": "~1.0.3", + "@lukeed/ms": "~2.0.2", "@microlink/mql": "~0.12.2", "@microlink/ping-url": "~1.4.11", "@microlink/ua": "~1.2.0", @@ -111,7 +112,6 @@ "is-email-like": "~1.0.0", "is-url-http": "~2.3.7", "lodash": "~4.17.21", - "ms": "~2.1.3", "on-finished": "~2.4.1", "p-any": "~3.0.0", "p-cancelable": "2.1.1", diff --git a/public/README.md b/public/README.md index 93d909d..e7b925e 100644 --- a/public/README.md +++ b/public/README.md @@ -26,7 +26,7 @@ So, no matter what type of query you use, **unavatar.io** has you covered. You c ### TTL -Type: `string`
+Type: `number`|`string`
Default: `'24h'`
Range: from `'1h'` to `'28d'` diff --git a/src/authentication.js b/src/authentication.js index e10c902..82ce126 100644 --- a/src/authentication.js +++ b/src/authentication.js @@ -1,7 +1,14 @@ 'use strict' +const debug = require('debug-logfmt')('unavatar:authentication') const { RateLimiterMemory } = require('rate-limiter-flexible') -const debug = require('debug-logfmt')('unavatar:rate') +const FrequencyCounter = require('frequency-counter') +const onFinished = require('on-finished') +const { format } = require('@lukeed/ms') + +const START = Date.now() +const reqsMin = new FrequencyCounter(60) +let reqs = 0 const { API_KEY, RATE_LIMIT_WINDOW, RATE_LIMIT } = require('./constant') @@ -36,20 +43,35 @@ const rateLimitError = (() => { })() module.exports = async (req, res, next) => { + ++reqs && reqsMin.inc() + onFinished(res, () => --reqs) + if (req.headers['x-api-key'] === API_KEY) return next() - const { msBeforeNext, remainingPoints: remaining } = await rateLimiter + const { msBeforeNext, remainingPoints: quotaRemaining } = await rateLimiter .consume(req.ipAddress) .catch(error => error) if (!res.writableEnded) { res.setHeader('X-Rate-Limit-Limit', RATE_LIMIT) - res.setHeader('X-Rate-Limit-Remaining', remaining) + res.setHeader('X-Rate-Limit-Remaining', quotaRemaining) res.setHeader('X-Rate-Limit-Reset', new Date(Date.now() + msBeforeNext)) - debug(req.ipAddress, { total: RATE_LIMIT, remaining }) + + const uptime = format(Date.now() - START) + const perMinute = reqsMin.freq() + const perSecond = Number(perMinute / 60).toFixed(1) + + debug(req.ipAddress, { + uptime, + reqs, + 'req/m': perMinute, + 'req/s': perSecond, + quotaTotal: RATE_LIMIT, + quotaRemaining + }) } - if (remaining) return next() + if (quotaRemaining) return next() res.setHeader('Retry-After', msBeforeNext / 1000) return next(rateLimitError) } diff --git a/src/avatar/resolve.js b/src/avatar/resolve.js index d97052f..35ba3e4 100644 --- a/src/avatar/resolve.js +++ b/src/avatar/resolve.js @@ -9,7 +9,6 @@ const isUrlHttp = require('is-url-http') const pTimeout = require('p-timeout') const pReflect = require('p-reflect') const { omit } = require('lodash') -const ms = require('ms') const isIterable = require('../util/is-iterable') @@ -40,7 +39,7 @@ const optimizeUrl = async (url, query) => { il: '', n: -1, w: AVATAR_SIZE, - ttl: ms(getTtl(ttl)), + ttl: getTtl(ttl), ...rest }).toString()}` } diff --git a/src/constant.js b/src/constant.js index 58e5450..4f19df0 100644 --- a/src/constant.js +++ b/src/constant.js @@ -1,7 +1,7 @@ 'use strict' +const { parse } = require('@lukeed/ms') const { existsSync } = require('fs') -const ms = require('ms') const TMP_FOLDER = existsSync('/dev/shm') ? '/dev/shm' : '/tmp' @@ -9,7 +9,9 @@ const { ALLOWED_REQ_HEADERS = ['accept-encoding', 'accept', 'user-agent'], AVATAR_SIZE = 400, AVATAR_TIMEOUT = 25000, - CACHE_TTL = ms('1y'), + TTL_DEFAULT = parse('1y'), + TTL_MIN = parse('1h'), + TTL_MAX = parse('28d'), NODE_ENV = 'development', PORT = 3000, RATE_LIMIT_WINDOW = 86400, @@ -26,7 +28,9 @@ module.exports = { ALLOWED_REQ_HEADERS, AVATAR_SIZE, AVATAR_TIMEOUT: Number(AVATAR_TIMEOUT), - CACHE_TTL, + TTL_DEFAULT, + TTL_MIN, + TTL_MAX, NODE_ENV, PORT, TMP_FOLDER, diff --git a/src/index.js b/src/index.js index 7a0da47..04eec01 100644 --- a/src/index.js +++ b/src/index.js @@ -5,6 +5,7 @@ const debug = require('debug-logfmt')('unavatar') const serveStatic = require('serve-static') const createRouter = require('router-http') const onFinished = require('on-finished') +const { format } = require('@lukeed/ms') const { forEach } = require('lodash') const send = require('send-http') const path = require('path') @@ -69,7 +70,7 @@ router debug( `${req.ipAddress} ${new URL(req.url, API_URL).toString()} ${ res.statusCode - } ${Math.round(req.timestamp())}ms` + } ${format(Math.round(req.timestamp()))}` ) }) next() diff --git a/src/send/cache.js b/src/send/cache.js index ead8185..c385df5 100644 --- a/src/send/cache.js +++ b/src/send/cache.js @@ -1,23 +1,22 @@ 'use strict' -const ms = require('ms') +const { parse } = require('@lukeed/ms') const memoize = require('../util/memoize') const send = require('.') -const { CACHE_TTL } = require('../constant') +const { TTL_DEFAULT, TTL_MIN, TTL_MAX } = require('../constant') const getTtl = memoize(ttl => { - if (ttl === undefined || ttl === null) return CACHE_TTL - const value = ms(ttl) - if (Number.isNaN(Number(value))) return CACHE_TTL - if (value < ms('1h') || value > ms('28d')) return CACHE_TTL + if (ttl === undefined || ttl === null) return TTL_DEFAULT + const value = typeof ttl === 'number' ? ttl : parse(ttl) + if (value === undefined || value < TTL_MIN || value > TTL_MAX) { return TTL_DEFAULT } return value }) module.exports = require('cacheable-response')({ logger: require('debug-logfmt')('cacheable-response'), - ttl: CACHE_TTL, + ttl: TTL_DEFAULT, staleTtl: false, get: async ({ req, res, fn }) => ({ ttl: getTtl(req.query.ttl), diff --git a/src/server.js b/src/server.js index 9b287d7..5b0e1aa 100644 --- a/src/server.js +++ b/src/server.js @@ -11,8 +11,6 @@ const { API_URL, NODE_ENV, PORT } = require('./constant') const server = createServer(require('.')) -require('./util/req-frequency')(server) - server.listen(PORT, () => { debug({ status: 'listening', @@ -24,6 +22,7 @@ server.listen(PORT, () => { process.on('uncaughtException', error => { debug.error('uncaughtException', { + message: error.message || error, requestUrl: error.response?.requestUrl }) }) diff --git a/src/util/cacheable-lookup.js b/src/util/cacheable-lookup.js index 4a28dba..56336ab 100644 --- a/src/util/cacheable-lookup.js +++ b/src/util/cacheable-lookup.js @@ -5,10 +5,10 @@ const Tangerine = require('tangerine') const { createMultiCache, createRedisCache } = require('./keyv') -const { CACHE_TTL } = require('../constant') +const { TTL_DEFAULT } = require('../constant') module.exports = new CacheableLookup({ - maxTtl: CACHE_TTL, + maxTtl: TTL_DEFAULT, cache: createMultiCache(createRedisCache({ namespace: 'dns' })), resolver: new Tangerine( { diff --git a/src/util/keyv.js b/src/util/keyv.js index 354ba2d..0a44d3b 100644 --- a/src/util/keyv.js +++ b/src/util/keyv.js @@ -8,12 +8,12 @@ const assert = require('assert') const redis = require('./redis') -const { CACHE_TTL } = require('../constant') +const { TTL_DEFAULT } = require('../constant') const createMultiCache = remote => new Keyv({ store: new KeyvMulti({ remote }) }) -const createKeyv = opts => new Keyv({ ttl: CACHE_TTL, ...opts }) +const createKeyv = opts => new Keyv({ ttl: TTL_DEFAULT, ...opts }) const createKeyvNamespace = opts => { assert(opts.namespace, '`opts.namespace` is required.') diff --git a/src/util/req-frequency.js b/src/util/req-frequency.js deleted file mode 100644 index be19817..0000000 --- a/src/util/req-frequency.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict' - -const debug = require('debug-logfmt')('req-frequency') -const FrequencyCounter = require('frequency-counter') -const onFinished = require('on-finished') - -module.exports = server => { - const min = new FrequencyCounter(60) - let requests = 0 - const info = () => { - const perMinute = min.freq() - return { requests, perMinute, perSecond: Number(perMinute / 60).toFixed(1) } - } - server.on('request', (_, res) => { - ++requests - min.inc() - debug(info()) - onFinished(res, () => --requests) - }) - return info -} diff --git a/test/query-parameters.js b/test/query-parameters.js index e149c26..a01a659 100644 --- a/test/query-parameters.js +++ b/test/query-parameters.js @@ -1,10 +1,10 @@ 'use strict' +const { parse } = require('@lukeed/ms') const test = require('ava') const got = require('got') -const ms = require('ms') -const { CACHE_TTL } = require('../src/constant') +const { TTL_DEFAULT } = require('../src/constant') const { runServer } = require('./helpers') const { getTtl } = require('../src/send/cache') @@ -52,13 +52,14 @@ test('fallback # use default value if fallback provided is not reachable', async }) test('ttl', t => { - t.is(getTtl(), CACHE_TTL) - t.is(getTtl(null), CACHE_TTL) - t.is(getTtl(undefined), CACHE_TTL) - t.is(getTtl(0), CACHE_TTL) - t.is(getTtl('foo'), CACHE_TTL) - t.is(getTtl('29d'), CACHE_TTL) - t.is(getTtl('29d'), CACHE_TTL) - t.is(getTtl(ms('2h')), CACHE_TTL) - t.is(getTtl('2h'), ms('2h')) + t.is(getTtl(), TTL_DEFAULT) + t.is(getTtl(''), TTL_DEFAULT) + t.is(getTtl(null), TTL_DEFAULT) + t.is(getTtl(undefined), TTL_DEFAULT) + t.is(getTtl(0), TTL_DEFAULT) + t.is(getTtl('foo'), TTL_DEFAULT) + t.is(getTtl('29d'), TTL_DEFAULT) + t.is(getTtl('29d'), TTL_DEFAULT) + t.is(getTtl(parse('2h')), parse('2h')) + t.is(getTtl('2h'), parse('2h')) })