Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Continue file upload #1012

Merged
merged 10 commits into from
May 29, 2024
50 changes: 39 additions & 11 deletions js/hatrac.js
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,10 @@ var ERMrest = (function(module) {
this.uploadingSize = 0;

this.chunks = [];
// array of true values for tracking which chunks are uploaded so far
// used to determine what chunk to resume upload from if needed
this.chunkTracker = [];
this.startChunkIdx = 0;

this.log = console.log;

Expand Down Expand Up @@ -409,9 +413,13 @@ var ERMrest = (function(module) {
* @desc Call this function to determine file exists on the server
* If it doesn't then resolve the promise with url.
* If it does then set isPaused, completed and jobDone to true
* @param {string} jobUrl - if an existing job is being tracked locally and the checksum for current `upload`
* matches that matched job, return the stored previousJobUrl to be used if a 409 is returned
* - a 409 could mean the namespace already exists and we have an existing job for that namespace we know is partially uplaoded
* - if all the above is true, set the `upload.chunkUrl` to the jobUrl we were tracking locally
* @returns {Promise}
*/
upload.prototype.fileExists = function(contextHeaderParams) {
upload.prototype.fileExists = function(previousJobUrl, contextHeaderParams) {
var self = this;

var deferred = module._q.defer();
Expand Down Expand Up @@ -480,10 +488,16 @@ var ERMrest = (function(module) {
// 403 - file exists but user can't read it -> create a new one
// 404 - file doesn't exist -> create new one
// 409 - The parent path does not denote a namespace OR the namespace already exists (from hatrac docs)
if (response.status == 403 || response.status == 404 || response.status == 409) {
deferred.resolve(self.url);
if (response.status == 403 || response.status == 404) {
deferred.resolve(self.url);
} else if (response.status == 409) {
// the namespace might exist with no content, maybe there is a partial upload
// set the chunkUrl to the previousJobUrl that we stored in chaise with a partial upload
// previousJobUrl = self.url + ';upload/' + job.hash
if (previousJobUrl) self.chunkUrl = previousJobUrl;
deferred.resolve(self.url);
} else {
deferred.reject(module.responseToError(response));
deferred.reject(module.responseToError(response));
}
});

Expand Down Expand Up @@ -568,11 +582,12 @@ var ERMrest = (function(module) {
* If the completed flag is true, then this means that all chunks were already uploaded, thus it will resolve the promize with url
* else it will start uploading the chunks. If the job was paused then resume by uploading just those chunks which were not completed.
*
* @param {number} startChunkIdx - the index of the chunk to start uploading from in case of resuming a found incomplete upload job
* @returns {Promise} A promise resolved with a url where we uploaded the file
* or rejected with error if unable to upload any chunk
* and notified with a progress handler, sending number in bytes uploaded uptil now
*/
upload.prototype.start = function() {
upload.prototype.start = function(startChunkIdx) {
var self = this;

this.erred = false;
Expand Down Expand Up @@ -610,6 +625,13 @@ var ERMrest = (function(module) {
this.chunks.push(chunk);
start = end;
}

this.startChunkIdx = startChunkIdx || 0;
// intialize array to the same length as the number of chunks we have
// this initializes every index to `Empty`
this.chunkTracker = Array(this.chunks.length);
// set index in array to true for each chunk we know is already uploaded
for (var j = 0; j < startChunkIdx; j++) this.chunkTracker[j] = true;
}
}

Expand All @@ -618,7 +640,10 @@ var ERMrest = (function(module) {

this.chunkQueue = [];

this.chunks.forEach(function(chunk) {
this.chunks.forEach(function(chunk, idx) {
// check the startChunkIdx before uploading the chunk in the case we are resuming an upload job
if (idx < startChunkIdx) return;

chunk.retryCount = 0;
self.chunkQueue.push(chunk);
part++;
Expand Down Expand Up @@ -913,9 +938,9 @@ var ERMrest = (function(module) {
var config = {
headers: _generateContextHeader(contextHeaderParams)
};
// If chunkUrl is not null then fetch the status of the uploadJob
// If chunkUrl is not null then fetch the status of the uploadJob
// and resolve promise with it.
// else just resolved it without any response
// else resolve it without any response
if (this.chunkUrl) {
this.http.get(this._getAbsoluteUrl(this.chunkUrl), config).then(function(response) {
deferred.resolve(response);
Expand Down Expand Up @@ -956,13 +981,14 @@ var ERMrest = (function(module) {
* @private
* @desc This function should be called to update the progress of upload
* It calls the onProgressChanged callback that the user subscribes
* In addition if the upload has been combleted then it will call onUploadCompleted for regular upload
* In addition if the upload has been completed then it will call onUploadCompleted for regular upload
* and completeUpload to complete the chunk upload
*/
upload.prototype._updateProgressBar = function() {
var length = this.chunks.length;
var progressDone = 0;
var chunksComplete = 0;
// progressDone and chunksComplete should be intiialized if we had an existing upload job
var progressDone = this.startChunkIdx * this.PART_SIZE;
var chunksComplete = this.startChunkIdx;
for (var i = 0; i < length; i++) {
progressDone = progressDone + this.chunks[i].progress;
if (this.chunks[i].completed) chunksComplete++;
Expand Down Expand Up @@ -1130,6 +1156,8 @@ var ERMrest = (function(module) {

deferred.resolve();

// this chunk was successfully uploaded, update the chunkTracker
upload.chunkTracker[self.index] = true;
upload._updateProgressBar();
}, function(response) {
self.progress = 0;
Expand Down