allow only partial + limit chunck to 99mb

src/bundle/partial.ts

@@ -6,7 +6,7 @@ import { buffer as readBuffer } from 'node:stream/consumers'
 import { createBrotliCompress } from 'node:zlib'
 import { log, spinner as spinnerC } from '@clack/prompts'
 import * as tus from 'tus-js-client'
-import { generateManifest, getLocalConfig, sendEvent } from '../utils'
+import { generateManifest, getLocalConfig, MAX_CHUNK_SIZE, sendEvent } from '../utils'
 
 export async function prepareBundlePartialFiles(path: string, apikey: string, orgId: string, appid: string) {
   const spinner = spinnerC()
@@ -39,7 +39,7 @@ function convertToUnixPath(windowsPath: string): string {
   return normalizedPath.split(win32.sep).join(posix.sep)
 }
 
-export async function uploadPartial(apikey: string, manifest: manifestType, path: string, appId: string, name: string, orgId: string): Promise<any[] | null> {
+export async function uploadPartial(apikey: string, manifest: manifestType, path: string, appId: string, name: string, orgId: string, chunkSize?: number): Promise<any[] | null> {
   const spinner = spinnerC()
   spinner.start('Preparing partial update with TUS protocol')
   const startTime = performance.now()
@@ -57,6 +57,7 @@ export async function uploadPartial(apikey: string, manifest: manifestType, path
     return new Promise((resolve, reject) => {
       const upload = new tus.Upload(fileBuffer as any, {
         endpoint: `${localConfig.hostFilesApi}/files/upload/attachments/`,
+        chunkSize: chunkSize || MAX_CHUNK_SIZE,
         metadata: {
           filename: `orgs/${orgId}/apps/${appId}/${name}/${filePathUnix}`,
         },

src/bundle/upload.ts

@@ -15,7 +15,7 @@ import { checkAppExistsAndHasPermissionOrgErr } from '../api/app'
 import { encryptSource } from '../api/crypto'
 import { encryptChecksumV2, encryptSourceV2 } from '../api/cryptoV2'
 import { checkAlerts } from '../api/update'
-import { ALERT_MB, baseKeyPub, baseKeyV2, checkChecksum, checkCompatibility, checkPlanValid, convertAppName, createSupabaseClient, deletedFailedVersion, findSavedKey, formatError, getAppId, getConfig, getLocalConfig, getLocalDepenencies, getOrganizationId, getPMAndCommand, getRemoteFileConfig, hasOrganizationPerm, OrganizationPerm, readPackageJson, regexSemver, sendEvent, updateConfig, updateOrCreateChannel, updateOrCreateVersion, UPLOAD_TIMEOUT, uploadTUS, uploadUrl, verifyUser, zipFile } from '../utils'
+import { ALERT_MB, baseKeyPub, baseKeyV2, checkChecksum, checkCompatibility, checkPlanValid, convertAppName, createSupabaseClient, deletedFailedVersion, findSavedKey, formatError, getAppId, getConfig, getLocalConfig, getLocalDepenencies, getOrganizationId, getPMAndCommand, getRemoteFileConfig, hasOrganizationPerm, MAX_CHUNK_SIZE, OrganizationPerm, readPackageJson, regexSemver, sendEvent, updateConfig, updateOrCreateChannel, updateOrCreateVersion, UPLOAD_TIMEOUT, uploadTUS, uploadUrl, verifyUser, zipFile } from '../utils'
 import { checkIndexPosition, searchInDirectory } from './check'
 import { prepareBundlePartialFiles, uploadPartial } from './partial'
 
@@ -48,6 +48,7 @@ interface Options extends OptionsBase {
   timeout?: number
   multipart?: boolean
   partial?: boolean
+  partialOnly?: boolean
   tus?: boolean
   encryptedChecksum?: string
   packageJson?: string
@@ -584,7 +585,7 @@ export async function uploadBundle(preAppid: string, options: Options, shouldExi
     options.partial = false
   }
   else {
-    options.partial = options.partial || fileConfig.partialUploadForced
+    options.partial = options.partial || options.partialOnly || fileConfig.partialUploadForced
   }
 
   const manifest: manifestType = options.partial ? await prepareBundlePartialFiles(path, apikey, orgId, appid) : []
@@ -594,6 +595,10 @@ export async function uploadBundle(preAppid: string, options: Options, shouldExi
     log.error(`Cannot add bundle ${formatError(dbError)}`)
     program.error('')
   }
+  if (options.tusChunkSize && options.tusChunkSize > MAX_CHUNK_SIZE) {
+    log.error(`Chunk size ${options.tusChunkSize} is greater than the maximum chunk size ${MAX_CHUNK_SIZE}, using the maximum chunk size`)
+    options.tusChunkSize = MAX_CHUNK_SIZE
+  }
 
   if (zipped && (s3BucketName || s3Endpoint || s3Region || s3Apikey || s3Apisecret || s3Port || s3SSL)) {
     if (!s3BucketName || !s3Endpoint || !s3Region || !s3Apikey || !s3Apisecret || !s3Port) {
@@ -618,14 +623,16 @@ export async function uploadBundle(preAppid: string, options: Options, shouldExi
     versionData.storage_provider = 'external'
   }
   else if (zipped) {
-    await uploadBundleToCapgoCloud(apikey, supabase, appid, bundle, orgId, zipped, options)
+    if (!options.partialOnly) {
+      await uploadBundleToCapgoCloud(apikey, supabase, appid, bundle, orgId, zipped, options)
+    }
 
     let finalManifest: Awaited<ReturnType<typeof uploadPartial>> | null = null
     try {
       if (options.dryUpload) {
         options.partial = false
       }
-      finalManifest = options.partial ? await uploadPartial(apikey, manifest, path, appid, bundle, orgId) : null
+      finalManifest = options.partial ? await uploadPartial(apikey, manifest, path, appid, bundle, orgId, options.tusChunkSize) : null
     }
     catch (err) {
       log.info(`Failed to upload partial files to capgo cloud. Error: ${formatError(err)}. This is not a critical error, the bundle has been uploaded without the partial files`)

src/index.ts

@@ -156,6 +156,7 @@ bundle
   .option('--tus', 'Upload the bundle using TUS to Capgo cloud')
   .option('--tus-chunk-size <tusChunkSize>', 'Chunk size for the TUS upload')
   .option('--partial', 'Upload partial files to Capgo cloud')
+  .option('--partial-only', 'Upload only partial files to Capgo cloud, skip the zipped file, useful for big bundle')
   .option('--encrypted-checksum <encryptedChecksum>', 'An encrypted checksum (signature). Used only when uploading an external bundle.')
   .option('--auto-set-bundle', 'Set the bundle in capacitor.config.json')
   .option('--dry-upload', 'Dry upload the bundle process, mean it will not upload the files but add the row in database (useful for testing)')

src/utils.ts

@@ -29,6 +29,8 @@ export const defaultApiHost = 'https://api.capgo.app'
 export const defaultHostWeb = 'https://web.capgo.app'
 export const ALERT_MB = 20
 export const UPLOAD_TIMEOUT = 120000
+export const MAX_CHUNK_SIZE = 1024 * 1024 * 99 // 99MB
+
 const PACKNAME = 'package.json'
 
 export type ArrayElement<ArrayType extends readonly unknown[]> =
@@ -787,7 +789,7 @@ export async function uploadTUS(apikey: string, data: Buffer, orgId: string, app
     const upload = new tus.Upload(data as any, {
       endpoint: `${localConfig.hostFilesApi}/files/upload/attachments/`,
       // parallelUploads: multipart,
-      chunkSize: chunkSize || Number.POSITIVE_INFINITY,
+      chunkSize: chunkSize || MAX_CHUNK_SIZE,
       metadataForPartialUploads: {
         filename: `orgs/${orgId}/apps/${appId}/${name}.zip`,
         filetype: 'application/gzip',