feat: use axios and retries

subsquid · Mar 9, 2024 · 6f833e3 · 6f833e3
1 parent 4c416c6
commit 6f833e3
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 462 deletions.
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@subsquid/cli",
   "description": "squid cli tool",
-  "version": "2.9.0-beta.1",
+  "version": "2.9.0-beta.2",
   "license": "GPL-3.0-or-later",
   "repository": "[email protected]:subsquid/squid-cli.git",
   "publishConfig": {
@@ -76,6 +76,8 @@
     "@types/lodash": "^4.14.202",
     "@types/targz": "^1.0.4",
     "async-retry": "^1.3.3",
+    "axios": "^1.6.7",
+    "axios-retry": "^4.0.0",
     "blessed-contrib": "^4.11.0",
     "chalk": "^4.1.2",
     "cli-select": "^1.1.2",
@@ -92,7 +94,6 @@
     "lodash": "^4.17.21",
     "ms": "^2.1.3",
     "neo-blessed": "^0.2.0",
-    "node-fetch": "^2.6.7",
     "pretty-bytes": "^5.6.0",
     "qs": "^6.11.2",
     "reblessed": "^0.2.1",
@@ -114,7 +115,6 @@
     "@types/js-yaml": "^4.0.9",
     "@types/ms": "^0.7.34",
     "@types/node": "^20.11.17",
-    "@types/node-fetch": "^2.6.11",
     "@types/qs": "^6.9.11",
     "@types/split2": "^3.2.1",
     "@typescript-eslint/eslint-plugin": "^6.21.0",

diff --git a/src/api/api.ts b/src/api/api.ts
@@ -1,14 +1,23 @@
 import path from 'path';
 
+import axios, { Method } from 'axios';
+import axiosRetry, { IAxiosRetryConfig, isNetworkOrIdempotentRequestError } from 'axios-retry';
 import chalk from 'chalk';
 import { pickBy } from 'lodash';
-import fetch from 'node-fetch';
-import qs from 'qs';
+import ms from 'ms';
 
 import { getConfig } from '../config';
 
 const API_DEBUG = process.env.API_DEBUG === 'true';
 
+const DEFAULT_RETRY: IAxiosRetryConfig = {
+  retries: 10,
+  retryDelay: axiosRetry.exponentialDelay,
+  retryCondition: isNetworkOrIdempotentRequestError,
+};
+
+axiosRetry(axios, DEFAULT_RETRY);
+
 let version = 'unknown';
 try {
   // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -17,7 +26,7 @@ try {
 
 export class ApiError extends Error {
   constructor(
-    public status: number,
+    public request: { status: number; method: string; url: string },
     public body: {
       error: string;
       message?: string;
@@ -43,81 +52,78 @@ export async function api<T = any>({
   path,
   data,
   query = {},
+  headers = {},
   auth,
   responseType = 'json',
   abortController,
+  retry,
 }: {
-  method: 'get' | 'post' | 'put' | 'delete' | 'patch';
+  method: Method;
   path: string;
   query?: Record<string, string | string[] | boolean | number | undefined>;
   data?: unknown;
+  headers?: Record<string, string>;
   auth?: { apiUrl: string; credentials: string };
   responseType?: 'json' | 'stream';
   abortController?: AbortController;
+  retry?: number;
 }): Promise<{ body: T }> {
   const config = auth || getConfig();
 
-  const sanitizedQuery = pickBy(query, (v) => v);
-  const queryString = Object.keys(sanitizedQuery).length ? `?${qs.stringify(sanitizedQuery)}` : '';
-
-  const url = !path.startsWith('https') ? `${config.apiUrl}${path}${queryString}` : `${path}${queryString}`;
+  const started = Date.now();
+  // add the API_URL to the path if it's not a full url
+  const url = !path.startsWith('https') ? `${config.apiUrl}${path}` : path;
 
-  const headers = {
-    'Content-Type': 'application/json',
-    authorization: `token ${config.credentials}`,
+  const finalHeaders = {
+    authorization: url.startsWith(config.apiUrl) ? `token ${config.credentials}` : null,
     'X-CLI-Version': version,
+    ...headers,
   };
 
-  if (API_DEBUG) {
-    console.log(
-      chalk.dim(new Date().toISOString()),
-      chalk.cyan`[HTTP REQUEST]`,
-      chalk.dim(method?.toUpperCase()),
-      url,
-      chalk.dim(JSON.stringify({ headers })),
-    );
-    if (data) {
-      console.log(chalk.dim(JSON.stringify(data)));
-    }
-  }
-  const response = await fetch(url, {
+  const response = await axios(url, {
     method,
-    headers,
-    body: data ? JSON.stringify(data) : undefined,
+    headers: finalHeaders,
+    data,
     timeout: responseType === 'stream' ? 0 : undefined,
+    responseType,
+    params: pickBy(query, (v) => v),
     signal: abortController ? (abortController.signal as any) : undefined,
+    validateStatus: () => true,
+    'axios-retry': retry
+      ? {
+          ...DEFAULT_RETRY,
+          retries: retry,
+        }
+      : undefined,
   });
 
-  let body;
-  if (responseType === 'json') {
-    const rawBody = await response.text();
-    try {
-      body = responseType === 'json' ? JSON.parse(rawBody) : response.body;
-    } catch (e) {
-      body = rawBody;
-    }
-  } else {
-    body = response.body;
-  }
-
   if (API_DEBUG) {
     console.log(
       chalk.dim(new Date().toISOString()),
-      chalk.cyan`[HTTP RESPONSE]`,
-      url,
+      chalk.cyan`[${method.toUpperCase()}]`,
+      response.config.url,
       chalk.cyan(response.status),
+      ms(Date.now() - started),
       chalk.dim(JSON.stringify({ headers: response.headers })),
     );
-    if (body && responseType === 'json') {
-      console.log(chalk.dim(JSON.stringify(body, null, 2)));
+    if (response.data && responseType === 'json') {
+      console.log(chalk.dim(JSON.stringify(response.data)));
     }
   }
 
   switch (response.status) {
     case 200:
     case 201:
-      return { body };
+    case 204:
+      return { body: response.data };
     default:
-      throw new ApiError(response.status, body as any);
+      throw new ApiError(
+        {
+          method: method.toUpperCase(),
+          url: response.config.url || 'Unknown URL',
+          status: response.status,
+        },
+        response.data,
+      );
   }
 }
diff --git a/src/api/profile.ts b/src/api/profile.ts
@@ -22,7 +22,14 @@ export async function profile({
   });
 
   if (!body.payload) {
-    throw new ApiError(401, { error: 'username is missing' });
+    throw new ApiError(
+      {
+        status: 401,
+        method: 'get',
+        url: '/user',
+      },
+      { error: 'Credentials are missing or invalid' },
+    );
   }
 
   return body.payload;

diff --git a/src/api/upload.ts b/src/api/upload.ts
@@ -1,9 +1,8 @@
 import fs from 'fs';
 
 import FormData from 'form-data';
-import fetch from 'node-fetch';
 
-import { ApiError } from './api';
+import { api } from './api';
 import { getUploadUrl } from './squids';
 
 export async function uploadFile(orgCode: string, path: string): Promise<{ error: string | null; fileUrl?: string }> {
@@ -27,19 +26,14 @@ export async function uploadFile(orgCode: string, path: string): Promise<{ error
 
   body.append('file', fileStream, { knownLength: size });
 
-  const res = await fetch(uploadUrl, {
-    method: 'POST',
+  await api({
+    path: uploadUrl,
+    method: 'post',
     headers: {
       ...body.getHeaders(),
     },
-    body,
+    data: body,
   });
 
-  if (res.status !== 204) {
-    throw new ApiError(400, {
-      error: await res.text(),
-    });
-  }
-
   return { error: null, fileUrl };
 }
diff --git a/src/command.ts b/src/command.ts
@@ -30,10 +30,10 @@ export abstract class CliCommand extends Command {
   }
 
   async catch(error: any) {
-    const { status, body } = error;
+    const { request, body } = error;
 
     if (error instanceof ApiError) {
-      switch (status) {
+      switch (request.status) {
         case 401:
           return this.error(
             `Authentication failure. Please obtain a new deployment key at https://app.subsquid.io and follow the instructions`,
@@ -47,10 +47,10 @@ export abstract class CliCommand extends Command {
           }
           return this.error(body?.error || body?.message || `Validation error ${body}`);
         case 404:
+          const url = `${chalk.bold(request.method)} ${chalk.bold(request.url)}`;
+
           return this.error(
-            `Unknown API endpoint. Check that your are using the latest version of the Squid CLI. Message: ${
-              body?.error || body?.message || 'API url not found'
-            }`,
+            `Unknown API endpoint ${url}. Check that your are using the latest version of the Squid CLI. If the problem persists, please contact support.`,
           );
 
         case 405:
@@ -64,7 +64,7 @@ export abstract class CliCommand extends Command {
             [
               `Unknown network error occurred`,
               `==================`,
-              `Status: ${status}`,
+              `Status: ${request.status}`,
               `Body:\n${JSON.stringify(body)}`,
             ].join('\n'),
           );

diff --git a/src/commands/deploy.ts b/src/commands/deploy.ts
@@ -1,5 +1,5 @@
-import fs from 'fs';
-import path from 'path';
+import fs from 'node:fs';
+import path from 'node:path';
 import { promisify } from 'util';
 
 import { Args, Flags, ux as CliUx } from '@oclif/core';
@@ -8,6 +8,7 @@ import chalk from 'chalk';
 import { globSync } from 'glob';
 import ignore from 'ignore';
 import inquirer from 'inquirer';
+import prettyBytes from 'pretty-bytes';
 import targz from 'targz';
 
 import { deploySquid, uploadFile } from '../api';
@@ -174,9 +175,9 @@ export default class Deploy extends DeployCommand {
   private async pack({ buildDir, squidDir, archiveName }: { buildDir: string; squidDir: string; archiveName: string }) {
     CliUx.ux.action.start(`◷ Compressing the squid to ${archiveName} `);
 
-    const squidignore = createSquidIgnore(squidDir);
+    const squidIgnore = createSquidIgnore(squidDir);
 
-    if (!hasPackageJson(squidDir) || squidignore?.ignores(PACKAGE_JSON)) {
+    if (!hasPackageJson(squidDir) || squidIgnore?.ignores(PACKAGE_JSON)) {
       return this.showError(
         [
           `The ${PACKAGE_JSON} file was not found in the squid directory`,
@@ -211,41 +212,18 @@ export default class Deploy extends DeployCommand {
       src: squidDir,
       dest: squidArtifact,
       tar: {
-        // if squidignore does not exist, we fallback to the old ignore approach
-        ignore: squidignore
-          ? (name) => {
-              const relativePath = path.relative(path.resolve(squidDir), path.resolve(name));
-
-              if (squidignore.ignores(relativePath)) {
-                this.log(chalk.dim(`-- ignoring ${relativePath}`));
-                return true;
-              } else {
-                this.log(chalk.dim(`adding ${relativePath}`));
-                filesCount++;
-                return false;
-              }
-            }
-          : (name) => {
-              const relativePath = path.relative(path.resolve(squidDir), path.resolve(name));
-
-              switch (relativePath) {
-                case 'node_modules':
-                case 'builds':
-                case 'lib':
-                case 'Dockerfile':
-                // FIXME: .env ?
-                case '.git':
-                case '.github':
-                case '.idea':
-                  this.log(chalk.dim(`-- ignoring ${relativePath}`));
-                  return true;
-                default:
-                  this.log(chalk.dim(`adding ${relativePath}`));
-
-                  filesCount++;
-                  return false;
-              }
-            },
+        ignore: (name) => {
+          const relativePath = path.relative(path.resolve(squidDir), path.resolve(name));
+
+          if (squidIgnore.ignores(relativePath)) {
+            this.log(chalk.dim(`-- ignoring ${relativePath}`));
+            return true;
+          } else {
+            this.log(chalk.dim(`adding ${relativePath}`));
+            filesCount++;
+            return false;
+          }
+        },
       },
     });
 
@@ -256,7 +234,9 @@ export default class Deploy extends DeployCommand {
       );
     }
 
-    CliUx.ux.action.stop(`${filesCount} file(s) ✔️`);
+    const squidArtifactStats = fs.statSync(squidArtifact);
+
+    CliUx.ux.action.stop(`${filesCount} files, ${prettyBytes(squidArtifactStats.size)} ✔️`);
 
     return squidArtifact;
   }
@@ -290,16 +270,27 @@ function hasLockFile(squidDir: string, lockFile?: string) {
 }
 
 function createSquidIgnore(squidDir: string) {
-  const ig = ignore();
+  const ig = ignore().add(
+    // default ignore patterns
+    ['node_modules', '.git'],
+  );
 
   const ignoreFilePaths = globSync(['.squidignore', '**/.squidignore'], {
     cwd: squidDir,
     nodir: true,
     posix: true,
   });
 
-  if (ignoreFilePaths.length === 0) {
-    return undefined;
+  if (!ignoreFilePaths.length) {
+    return ig.add([
+      // squid uploaded archives directory
+      '/builds',
+      // squid built files
+      '/lib',
+      // IDE files
+      '.idea',
+      '.vscode',
+    ]);
   }
 
   for (const ignoreFilePath of ignoreFilePaths) {