Merge pull request #307 from microlinkhq/next

refactor: ttl layer

package.json

@@ -88,6 +88,7 @@
     "@keyvhq/multi": "~2.1.0",
     "@keyvhq/redis": "~2.1.0",
     "@kikobeats/time-span": "~1.0.3",
+    "@lukeed/ms": "~2.0.2",
     "@microlink/mql": "~0.12.2",
     "@microlink/ping-url": "~1.4.11",
     "@microlink/ua": "~1.2.0",
@@ -111,7 +112,6 @@
     "is-email-like": "~1.0.0",
     "is-url-http": "~2.3.7",
     "lodash": "~4.17.21",
-    "ms": "~2.1.3",
     "on-finished": "~2.4.1",
     "p-any": "~3.0.0",
     "p-cancelable": "2.1.1",

public/README.md

@@ -26,7 +26,7 @@ So, no matter what type of query you use, **unavatar.io** has you covered. You c
 
 ### TTL
 
-Type: `string`<br/>
+Type: `number`|`string`<br/>
 Default: `'24h'`<br/>
 Range: from `'1h'` to `'28d'`
 

src/authentication.js

@@ -1,7 +1,14 @@
 'use strict'
 
+const debug = require('debug-logfmt')('unavatar:authentication')
 const { RateLimiterMemory } = require('rate-limiter-flexible')
-const debug = require('debug-logfmt')('unavatar:rate')
+const FrequencyCounter = require('frequency-counter')
+const onFinished = require('on-finished')
+const { format } = require('@lukeed/ms')
+
+const START = Date.now()
+const reqsMin = new FrequencyCounter(60)
+let reqs = 0
 
 const { API_KEY, RATE_LIMIT_WINDOW, RATE_LIMIT } = require('./constant')
 
@@ -36,20 +43,35 @@ const rateLimitError = (() => {
 })()
 
 module.exports = async (req, res, next) => {
+  ++reqs && reqsMin.inc()
+  onFinished(res, () => --reqs)
+
   if (req.headers['x-api-key'] === API_KEY) return next()
 
-  const { msBeforeNext, remainingPoints: remaining } = await rateLimiter
+  const { msBeforeNext, remainingPoints: quotaRemaining } = await rateLimiter
     .consume(req.ipAddress)
     .catch(error => error)
 
   if (!res.writableEnded) {
     res.setHeader('X-Rate-Limit-Limit', RATE_LIMIT)
-    res.setHeader('X-Rate-Limit-Remaining', remaining)
+    res.setHeader('X-Rate-Limit-Remaining', quotaRemaining)
     res.setHeader('X-Rate-Limit-Reset', new Date(Date.now() + msBeforeNext))
-    debug(req.ipAddress, { total: RATE_LIMIT, remaining })
+
+    const uptime = format(Date.now() - START)
+    const perMinute = reqsMin.freq()
+    const perSecond = Number(perMinute / 60).toFixed(1)
+
+    debug(req.ipAddress, {
+      uptime,
+      reqs,
+      'req/m': perMinute,
+      'req/s': perSecond,
+      quotaTotal: RATE_LIMIT,
+      quotaRemaining
+    })
   }
 
-  if (remaining) return next()
+  if (quotaRemaining) return next()
   res.setHeader('Retry-After', msBeforeNext / 1000)
   return next(rateLimitError)
 }

src/avatar/resolve.js

@@ -9,7 +9,6 @@ const isUrlHttp = require('is-url-http')
 const pTimeout = require('p-timeout')
 const pReflect = require('p-reflect')
 const { omit } = require('lodash')
-const ms = require('ms')
 
 const isIterable = require('../util/is-iterable')
 
@@ -40,7 +39,7 @@ const optimizeUrl = async (url, query) => {
     il: '',
     n: -1,
     w: AVATAR_SIZE,
-    ttl: ms(getTtl(ttl)),
+    ttl: getTtl(ttl),
     ...rest
   }).toString()}`
 }

src/constant.js

@@ -1,15 +1,17 @@
 'use strict'
 
+const { parse } = require('@lukeed/ms')
 const { existsSync } = require('fs')
-const ms = require('ms')
 
 const TMP_FOLDER = existsSync('/dev/shm') ? '/dev/shm' : '/tmp'
 
 const {
   ALLOWED_REQ_HEADERS = ['accept-encoding', 'accept', 'user-agent'],
   AVATAR_SIZE = 400,
   AVATAR_TIMEOUT = 25000,
-  CACHE_TTL = ms('1y'),
+  TTL_DEFAULT = parse('1y'),
+  TTL_MIN = parse('1h'),
+  TTL_MAX = parse('28d'),
   NODE_ENV = 'development',
   PORT = 3000,
   RATE_LIMIT_WINDOW = 86400,
@@ -26,7 +28,9 @@ module.exports = {
   ALLOWED_REQ_HEADERS,
   AVATAR_SIZE,
   AVATAR_TIMEOUT: Number(AVATAR_TIMEOUT),
-  CACHE_TTL,
+  TTL_DEFAULT,
+  TTL_MIN,
+  TTL_MAX,
   NODE_ENV,
   PORT,
   TMP_FOLDER,

src/index.js

@@ -5,6 +5,7 @@ const debug = require('debug-logfmt')('unavatar')
 const serveStatic = require('serve-static')
 const createRouter = require('router-http')
 const onFinished = require('on-finished')
+const { format } = require('@lukeed/ms')
 const { forEach } = require('lodash')
 const send = require('send-http')
 const path = require('path')
@@ -69,7 +70,7 @@ router
         debug(
           `${req.ipAddress} ${new URL(req.url, API_URL).toString()} ${
             res.statusCode
-          } ${Math.round(req.timestamp())}ms`
+          } ${format(Math.round(req.timestamp()))}`
         )
       })
       next()

src/send/cache.js

@@ -1,23 +1,22 @@
 'use strict'
 
-const ms = require('ms')
+const { parse } = require('@lukeed/ms')
 
 const memoize = require('../util/memoize')
 const send = require('.')
 
-const { CACHE_TTL } = require('../constant')
+const { TTL_DEFAULT, TTL_MIN, TTL_MAX } = require('../constant')
 
 const getTtl = memoize(ttl => {
-  if (ttl === undefined || ttl === null) return CACHE_TTL
-  const value = ms(ttl)
-  if (Number.isNaN(Number(value))) return CACHE_TTL
-  if (value < ms('1h') || value > ms('28d')) return CACHE_TTL
+  if (ttl === undefined || ttl === null) return TTL_DEFAULT
+  const value = typeof ttl === 'number' ? ttl : parse(ttl)
+  if (value === undefined || value < TTL_MIN || value > TTL_MAX) { return TTL_DEFAULT }
   return value
 })
 
 module.exports = require('cacheable-response')({
   logger: require('debug-logfmt')('cacheable-response'),
-  ttl: CACHE_TTL,
+  ttl: TTL_DEFAULT,
   staleTtl: false,
   get: async ({ req, res, fn }) => ({
     ttl: getTtl(req.query.ttl),

src/server.js

@@ -11,8 +11,6 @@ const { API_URL, NODE_ENV, PORT } = require('./constant')
 
 const server = createServer(require('.'))
 
-require('./util/req-frequency')(server)
-
 server.listen(PORT, () => {
   debug({
     status: 'listening',
@@ -24,6 +22,7 @@ server.listen(PORT, () => {
 
 process.on('uncaughtException', error => {
   debug.error('uncaughtException', {
+    message: error.message || error,
     requestUrl: error.response?.requestUrl
   })
 })

src/util/cacheable-lookup.js

@@ -5,10 +5,10 @@ const Tangerine = require('tangerine')
 
 const { createMultiCache, createRedisCache } = require('./keyv')
 
-const { CACHE_TTL } = require('../constant')
+const { TTL_DEFAULT } = require('../constant')
 
 module.exports = new CacheableLookup({
-  maxTtl: CACHE_TTL,
+  maxTtl: TTL_DEFAULT,
   cache: createMultiCache(createRedisCache({ namespace: 'dns' })),
   resolver: new Tangerine(
     {

src/util/keyv.js

@@ -8,12 +8,12 @@ const assert = require('assert')
 
 const redis = require('./redis')
 
-const { CACHE_TTL } = require('../constant')
+const { TTL_DEFAULT } = require('../constant')
 
 const createMultiCache = remote =>
   new Keyv({ store: new KeyvMulti({ remote }) })
 
-const createKeyv = opts => new Keyv({ ttl: CACHE_TTL, ...opts })
+const createKeyv = opts => new Keyv({ ttl: TTL_DEFAULT, ...opts })
 
 const createKeyvNamespace = opts => {
   assert(opts.namespace, '`opts.namespace` is required.')

test/query-parameters.js

@@ -1,10 +1,10 @@
 'use strict'
 
+const { parse } = require('@lukeed/ms')
 const test = require('ava')
 const got = require('got')
-const ms = require('ms')
 
-const { CACHE_TTL } = require('../src/constant')
+const { TTL_DEFAULT } = require('../src/constant')
 
 const { runServer } = require('./helpers')
 const { getTtl } = require('../src/send/cache')
@@ -52,13 +52,14 @@ test('fallback # use default value if fallback provided is not reachable', async
 })
 
 test('ttl', t => {
-  t.is(getTtl(), CACHE_TTL)
-  t.is(getTtl(null), CACHE_TTL)
-  t.is(getTtl(undefined), CACHE_TTL)
-  t.is(getTtl(0), CACHE_TTL)
-  t.is(getTtl('foo'), CACHE_TTL)
-  t.is(getTtl('29d'), CACHE_TTL)
-  t.is(getTtl('29d'), CACHE_TTL)
-  t.is(getTtl(ms('2h')), CACHE_TTL)
-  t.is(getTtl('2h'), ms('2h'))
+  t.is(getTtl(), TTL_DEFAULT)
+  t.is(getTtl(''), TTL_DEFAULT)
+  t.is(getTtl(null), TTL_DEFAULT)
+  t.is(getTtl(undefined), TTL_DEFAULT)
+  t.is(getTtl(0), TTL_DEFAULT)
+  t.is(getTtl('foo'), TTL_DEFAULT)
+  t.is(getTtl('29d'), TTL_DEFAULT)
+  t.is(getTtl('29d'), TTL_DEFAULT)
+  t.is(getTtl(parse('2h')), parse('2h'))
+  t.is(getTtl('2h'), parse('2h'))
 })