From 0c73bcc771eced72fb3378a05d0c0bf4236056a8 Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Fri, 3 May 2024 17:42:07 -0700 Subject: [PATCH 1/6] in progress changes to support continuing a previous upload job that was interrupted but not canceled/deleted --- js/hatrac.js | 39 ++++++++++++++++++++++++++++----------- js/http.js | 3 ++- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/js/hatrac.js b/js/hatrac.js index 6be32b5f..6521d7c7 100644 --- a/js/hatrac.js +++ b/js/hatrac.js @@ -320,6 +320,7 @@ var ERMrest = (function(module) { this.uploadingSize = 0; this.chunks = []; + this.chunkTracker = []; this.log = console.log; @@ -411,7 +412,7 @@ var ERMrest = (function(module) { * If it does then set isPaused, completed and jobDone to true * @returns {Promise} */ - upload.prototype.fileExists = function(contextHeaderParams) { + upload.prototype.fileExists = function(jobUrl, contextHeaderParams) { var self = this; var deferred = module._q.defer(); @@ -480,10 +481,16 @@ var ERMrest = (function(module) { // 403 - file exists but user can't read it -> create a new one // 404 - file doesn't exist -> create new one // 409 - The parent path does not denote a namespace OR the namespace already exists (from hatrac docs) - if (response.status == 403 || response.status == 404 || response.status == 409) { - deferred.resolve(self.url); + if (response.status == 403 || response.status == 404) { + deferred.resolve(self.url); + } else if (response.status == 409) { + // the namespace might exist with no content, maybe there is a partial upload + // set the chunkUrl to the jobUrl that we stored in chaise with a parital upload + // jobUrl = self.url + ';upload/' + job.hash + if (jobUrl) self.chunkUrl = jobUrl; + deferred.resolve(self.url); } else { - deferred.reject(module.responseToError(response)); + deferred.reject(module.responseToError(response)); } }); @@ -572,7 +579,7 @@ var ERMrest = (function(module) { * or rejected with error if unable to upload any chunk * and notified with a progress handler, sending number in bytes uploaded uptil now */ - upload.prototype.start = function() { + upload.prototype.start = function(startChunkIdx) { var self = this; this.erred = false; @@ -610,6 +617,10 @@ var ERMrest = (function(module) { this.chunks.push(chunk); start = end; } + + var areChunksUploaded = Array(this.chunks.length); + for (var j = 0; j < startChunkIdx; j++) areChunksUploaded[j] = true; + this.chunkTracker = areChunksUploaded; } } @@ -618,7 +629,9 @@ var ERMrest = (function(module) { this.chunkQueue = []; - this.chunks.forEach(function(chunk) { + this.chunks.forEach(function(chunk, idx) { + if (idx < startChunkIdx) return; + chunk.retryCount = 0; self.chunkQueue.push(chunk); part++; @@ -913,9 +926,9 @@ var ERMrest = (function(module) { var config = { headers: _generateContextHeader(contextHeaderParams) }; - // If chunkUrl is not null then fetch the status of the uploadJob + // If chunkUrl is not null then fetch the status of the uploadJob // and resolve promise with it. - // else just resolved it without any response + // else resolve it without any response if (this.chunkUrl) { this.http.get(this._getAbsoluteUrl(this.chunkUrl), config).then(function(response) { deferred.resolve(response); @@ -959,7 +972,11 @@ var ERMrest = (function(module) { * In addition if the upload has been combleted then it will call onUploadCompleted for regular upload * and completeUpload to complete the chunk upload */ - upload.prototype._updateProgressBar = function() { + upload.prototype._updateProgressBar = function(indexUploaded) { + if (indexUploaded !== undefined && indexUploaded !== null) { + this.chunkTracker[indexUploaded] = true; + } + var length = this.chunks.length; var progressDone = 0; var chunksComplete = 0; @@ -1064,7 +1081,7 @@ var ERMrest = (function(module) { deferred.resolve(); - upload._updateProgressBar(); + upload._updateProgressBar(this.index); return deferred.promise; } @@ -1130,7 +1147,7 @@ var ERMrest = (function(module) { deferred.resolve(); - upload._updateProgressBar(); + upload._updateProgressBar(self.index); }, function(response) { self.progress = 0; var status = response.status; diff --git a/js/http.js b/js/http.js index e2c6e8e8..32c343d1 100644 --- a/js/http.js +++ b/js/http.js @@ -48,7 +48,8 @@ * @private * @type {Number} */ - var _default_max_retries = 10; + // var _default_max_retries = 10; + var _default_max_retries = 2; /** * Initial timeout delay. This dely will be doubled each time to perform From ad827d9d004c03017ccacad070b3c8a1685d3404 Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Tue, 7 May 2024 10:48:53 -0700 Subject: [PATCH 2/6] keep track of chunk start index. Make sure we factor in startchunk idx for resuming upload jobs to ensure completion --- js/hatrac.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/js/hatrac.js b/js/hatrac.js index 6521d7c7..68257610 100644 --- a/js/hatrac.js +++ b/js/hatrac.js @@ -321,6 +321,7 @@ var ERMrest = (function(module) { this.chunks = []; this.chunkTracker = []; + this.startChunkIdx = 0; this.log = console.log; @@ -620,6 +621,7 @@ var ERMrest = (function(module) { var areChunksUploaded = Array(this.chunks.length); for (var j = 0; j < startChunkIdx; j++) areChunksUploaded[j] = true; + this.startChunkIdx = startChunkIdx; this.chunkTracker = areChunksUploaded; } } @@ -978,8 +980,9 @@ var ERMrest = (function(module) { } var length = this.chunks.length; - var progressDone = 0; - var chunksComplete = 0; + // progressDone and chunksComplete should be intiialized if we had an existing upload job + var progressDone = this.startChunkIdx * this.PART_SIZE; + var chunksComplete = this.startChunkIdx; for (var i = 0; i < length; i++) { progressDone = progressDone + this.chunks[i].progress; if (this.chunks[i].completed) chunksComplete++; From 7a15e45f64bd86c7e2c43259d9888a3e585f3a72 Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Tue, 7 May 2024 10:58:39 -0700 Subject: [PATCH 3/6] change retry count used during testing --- js/http.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/js/http.js b/js/http.js index 32c343d1..e2c6e8e8 100644 --- a/js/http.js +++ b/js/http.js @@ -48,8 +48,7 @@ * @private * @type {Number} */ - // var _default_max_retries = 10; - var _default_max_retries = 2; + var _default_max_retries = 10; /** * Initial timeout delay. This dely will be doubled each time to perform From 7ffdb91fbc673155085a35f45f9898b2376f48a2 Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Tue, 7 May 2024 11:30:52 -0700 Subject: [PATCH 4/6] add more comments and jsdoc info, change one implementation detail that didn't need to be conditional in updateProgressBar --- js/hatrac.js | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/js/hatrac.js b/js/hatrac.js index 68257610..dc34be6c 100644 --- a/js/hatrac.js +++ b/js/hatrac.js @@ -320,6 +320,8 @@ var ERMrest = (function(module) { this.uploadingSize = 0; this.chunks = []; + // array of true values for tracking which chunks are uploaded so far + // used to determine what chunk to resume upload from if needed this.chunkTracker = []; this.startChunkIdx = 0; @@ -411,6 +413,10 @@ var ERMrest = (function(module) { * @desc Call this function to determine file exists on the server * If it doesn't then resolve the promise with url. * If it does then set isPaused, completed and jobDone to true + * @param {string} jobUrl - if an existing job is being tracked locally and the checksum for current `upload` + * matches that matched job, send the stored jobUrl to be used if a 409 is returned + * - a 409 could mean the namespace already exists and we have an existing job for that namespace we know is partially uplaoded + * - if all the above is true, set the `upload.chunkUrl` to the jobUrl we were tracking locally * @returns {Promise} */ upload.prototype.fileExists = function(jobUrl, contextHeaderParams) { @@ -576,6 +582,7 @@ var ERMrest = (function(module) { * If the completed flag is true, then this means that all chunks were already uploaded, thus it will resolve the promize with url * else it will start uploading the chunks. If the job was paused then resume by uploading just those chunks which were not completed. * + * @param {number} startChunkIdx - the index of the chunk to start uploading from in case of resuming a found incomplete upload job * @returns {Promise} A promise resolved with a url where we uploaded the file * or rejected with error if unable to upload any chunk * and notified with a progress handler, sending number in bytes uploaded uptil now @@ -619,10 +626,9 @@ var ERMrest = (function(module) { start = end; } - var areChunksUploaded = Array(this.chunks.length); - for (var j = 0; j < startChunkIdx; j++) areChunksUploaded[j] = true; this.startChunkIdx = startChunkIdx; - this.chunkTracker = areChunksUploaded; + this.chunkTracker = Array(this.chunks.length); + for (var j = 0; j < startChunkIdx; j++) this.chunkTracker[j] = true; } } @@ -971,14 +977,10 @@ var ERMrest = (function(module) { * @private * @desc This function should be called to update the progress of upload * It calls the onProgressChanged callback that the user subscribes - * In addition if the upload has been combleted then it will call onUploadCompleted for regular upload + * In addition if the upload has been completed then it will call onUploadCompleted for regular upload * and completeUpload to complete the chunk upload */ - upload.prototype._updateProgressBar = function(indexUploaded) { - if (indexUploaded !== undefined && indexUploaded !== null) { - this.chunkTracker[indexUploaded] = true; - } - + upload.prototype._updateProgressBar = function() { var length = this.chunks.length; // progressDone and chunksComplete should be intiialized if we had an existing upload job var progressDone = this.startChunkIdx * this.PART_SIZE; @@ -1084,7 +1086,7 @@ var ERMrest = (function(module) { deferred.resolve(); - upload._updateProgressBar(this.index); + upload._updateProgressBar(); return deferred.promise; } @@ -1150,7 +1152,9 @@ var ERMrest = (function(module) { deferred.resolve(); - upload._updateProgressBar(self.index); + // this chunk was successfully uploaded, update the chunkTracker + this.chunkTracker[self.index] = true; + upload._updateProgressBar(); }, function(response) { self.progress = 0; var status = response.status; From 716c70be4f967fc89b0b2985ab6555aa303429c7 Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Wed, 8 May 2024 16:19:55 -0700 Subject: [PATCH 5/6] initalize chunkIdx if not provided. make sure to reference the right object for chunkTracker --- js/hatrac.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/hatrac.js b/js/hatrac.js index dc34be6c..bb4570b5 100644 --- a/js/hatrac.js +++ b/js/hatrac.js @@ -626,7 +626,7 @@ var ERMrest = (function(module) { start = end; } - this.startChunkIdx = startChunkIdx; + this.startChunkIdx = startChunkIdx || 0; this.chunkTracker = Array(this.chunks.length); for (var j = 0; j < startChunkIdx; j++) this.chunkTracker[j] = true; } @@ -1153,7 +1153,7 @@ var ERMrest = (function(module) { deferred.resolve(); // this chunk was successfully uploaded, update the chunkTracker - this.chunkTracker[self.index] = true; + upload.chunkTracker[self.index] = true; upload._updateProgressBar(); }, function(response) { self.progress = 0; From a6b7c033b84e6f26ddb863931247d835fd560d4e Mon Sep 17 00:00:00 2001 From: Joshua Chudy Date: Tue, 28 May 2024 18:27:00 -0700 Subject: [PATCH 6/6] update comments --- js/hatrac.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/js/hatrac.js b/js/hatrac.js index bb4570b5..7f72b243 100644 --- a/js/hatrac.js +++ b/js/hatrac.js @@ -414,12 +414,12 @@ var ERMrest = (function(module) { * If it doesn't then resolve the promise with url. * If it does then set isPaused, completed and jobDone to true * @param {string} jobUrl - if an existing job is being tracked locally and the checksum for current `upload` - * matches that matched job, send the stored jobUrl to be used if a 409 is returned + * matches that matched job, return the stored previousJobUrl to be used if a 409 is returned * - a 409 could mean the namespace already exists and we have an existing job for that namespace we know is partially uplaoded * - if all the above is true, set the `upload.chunkUrl` to the jobUrl we were tracking locally * @returns {Promise} */ - upload.prototype.fileExists = function(jobUrl, contextHeaderParams) { + upload.prototype.fileExists = function(previousJobUrl, contextHeaderParams) { var self = this; var deferred = module._q.defer(); @@ -492,9 +492,9 @@ var ERMrest = (function(module) { deferred.resolve(self.url); } else if (response.status == 409) { // the namespace might exist with no content, maybe there is a partial upload - // set the chunkUrl to the jobUrl that we stored in chaise with a parital upload - // jobUrl = self.url + ';upload/' + job.hash - if (jobUrl) self.chunkUrl = jobUrl; + // set the chunkUrl to the previousJobUrl that we stored in chaise with a partial upload + // previousJobUrl = self.url + ';upload/' + job.hash + if (previousJobUrl) self.chunkUrl = previousJobUrl; deferred.resolve(self.url); } else { deferred.reject(module.responseToError(response)); @@ -627,7 +627,10 @@ var ERMrest = (function(module) { } this.startChunkIdx = startChunkIdx || 0; + // intialize array to the same length as the number of chunks we have + // this initializes every index to `Empty` this.chunkTracker = Array(this.chunks.length); + // set index in array to true for each chunk we know is already uploaded for (var j = 0; j < startChunkIdx; j++) this.chunkTracker[j] = true; } } @@ -638,6 +641,7 @@ var ERMrest = (function(module) { this.chunkQueue = []; this.chunks.forEach(function(chunk, idx) { + // check the startChunkIdx before uploading the chunk in the case we are resuming an upload job if (idx < startChunkIdx) return; chunk.retryCount = 0;