Merge pull request #70 from codeforamerica/features/github-soft-rate-…

…limits

feat(github): implement handling of soft rate limits

crawler/lib/connections/github.js

@@ -1,8 +1,9 @@
 const axios = require('axios');
+const axiosRetryAfter = require('axios-retry-after');
 
 const { GITHUB_ACTOR: githubActor, GITHUB_TOKEN: githubToken } = process.env;
 
-module.exports = axios.create({
+const axiosClient = axios.create({
     baseURL: 'https://api.github.com',
     headers: {
         Accept: 'application/vnd.github.mercy-preview+json'
@@ -11,3 +12,21 @@ module.exports = axios.create({
         ? { username: githubActor, password: githubToken }
         : null
 });
+
+axiosClient.interceptors.response.use(null, axiosRetryAfter(axiosClient, {
+    // GitHub responds with 403 instead of 429 when soft rate limit is hit
+    isRetryable (error) {
+        return (
+            error.response && error.response.status === 403 &&
+            error.response.headers['retry-after']
+        );
+    },
+
+    wait (error) {
+        return new Promise(
+            resolve => setTimeout(resolve, error.response.headers['retry-after'] * 1000)
+        );
+    }
+}));
+
+module.exports = axiosClient;

crawler/package.json

@@ -5,7 +5,8 @@
   "author": "Chris Alfano <[email protected]>",
   "license": "Apache-2.0",
   "dependencies": {
-    "axios": "^0.19.2",
+    "axios": "^1.2.1",
+    "axios-retry-after": "^2.0.0",
     "csv-parser": "^2.3.3",
     "gitsheets": "^0.2.2",
     "parse-link-header": "^1.0.1",

docs/projects/crawler/contributing/crawler.md

@@ -4,6 +4,19 @@ The Crawler is built with Node.js and development is primarily supported for [Vi
 
 ## After Cloning or Pulling
 
+Make sure you're using node 14:
+
+```bash
+node --version
+```
+
+On Mac with node installed via Homebrew, you can switch your current shell to node 14:
+
+```bash
+brew install node@14
+export PATH="/usr/local/opt/node@14/bin:$PATH"
+```
+
 Install node modules:
 
 ```bash