S3 retries / v0.5.1

README.md

@@ -1,8 +1,29 @@
 # dvwebloader
 A web tool for uploading folders of files to a Dataverse dataset. See [the wiki](https://github.com/gdcc/dvwebloader/wiki) for further details.
 
+### Installation:
+
 The Hosted version at https://gdcc.github.io/dvwebloader can be used for testing. You should fork or install a local copy for production use (to avoid changes made in this repository immediately being available from your Dataverse installation.)
-You may also want to run the localinstall.sh script in the directory you download to to make and local copies of the libraries used.
+
+Example download/local install instructions assuming a web server serving the /var/www/html/ directory:
+
+    cd /var/www
+    git clone https://github.com/gdcc/dvwebloader.git
+    cd html
+    ln -s /var/www/dvwebloader/src /var/www/html/dvwebloader
+
+Future updates can be done by pulling the latest code, e.g.
+
+    cd /var/www/dvwebloader
+    git pull
+
+You may also want to run the localinstall.sh script in the directory the dvwebloader sourcecode is in to make and link local copies of the libraries used. (This should be done again if/when you update.)
+
+    cd /var/www/dvwebloader
+    chmod 755 localinstall.sh
+    cd src
+    ../localinstall.sh`
+
 
 ### Current integration mechanism (v5.13+):
 

src/dvwebloader.html

@@ -37,7 +37,7 @@ <h1><span id="title-text">Folder Upload</span></h1>
             </div>
             <div id="filelist"></div>
             <div id="credit">
-                <a href='https://github.com/gdcc/dvwebloader' target='_blank'>DVWebloader v0.5</a><span id="sponsor-text">, development sponsored by UiT/DataverseNO</span>
+                <a href='https://github.com/gdcc/dvwebloader' target='_blank'>DVWebloader v0.5.2</a><span id="sponsor-text">, development sponsored by UiT/DataverseNO with support from others.</span>
             </div>
         </main>
     </body>

src/js/fileupload2.js

@@ -364,7 +364,7 @@ async function fetchUploadLimits() {
     uploadLimits.successfullyRetrieved = false;
 
     try {
-        const uploadLimitsResponse = await $.ajax({
+        const uploadLimitsResponse = await ajaxWithRetry({
             url: siteUrl + '/api/datasets/:persistentId/uploadlimits?persistentId=' + datasetPid,
             headers: { "X-Dataverse-key": apiKey },
             type: 'GET',
@@ -455,7 +455,7 @@ async function retrieveDatasetInfo(isInitialLoad = true) {
         await fetchUploadLimits();
         updateUploadLimitsMessage();
         // First, check for dataset locks
-        const locksResponse = await $.ajax({
+        const locksResponse = await ajaxWithRetry({
             url: siteUrl + '/api/datasets/:persistentId/locks?persistentId=' + datasetPid,
             headers: { "X-Dataverse-key": apiKey },
             type: 'GET',
@@ -480,7 +480,7 @@ async function retrieveDatasetInfo(isInitialLoad = true) {
 
         // If locked InReview, check user permissions
         if (isLockedInReview) {
-            const permissionsResponse = await $.ajax({
+            const permissionsResponse = await ajaxWithRetry({
                 url: siteUrl + '/api/datasets/:persistentId/userPermissions?persistentId=' + datasetPid,
                 headers: { "X-Dataverse-key": apiKey },
                 type: 'GET',
@@ -499,7 +499,7 @@ async function retrieveDatasetInfo(isInitialLoad = true) {
         }
 
         // If not locked or user has permission, proceed with retrieving dataset info
-        const datasetResponse = await $.ajax({
+        const datasetResponse = await ajaxWithRetry({
             url: siteUrl + '/api/datasets/:persistentId/versions/:latest?persistentId=' + datasetPid,
             headers: { "X-Dataverse-key": apiKey },
             type: 'GET',
@@ -753,6 +753,7 @@ function sleep(ms) {
     return new Promise(resolve => setTimeout(resolve, ms));
 }
 
+
 async function cancelDatasetCreate() {
     //Page is going away - don't upload any more files, finish reporting current uploads, and then call cancelCreateCommand to clean up temp files
     if (directUploadEnabled) {
@@ -853,6 +854,79 @@ function updateUploadUrlCooldown(delayMs) {
   uploadUrlCooldownUntil = Math.max(uploadUrlCooldownUntil, Date.now() + delayMs);
 }
 
+/**
+ * A wrapper for $.ajax that retries on 50x errors (for S3) or 429 errors (for Dataset API).
+ * @param {Object} options - The same options object as passed to $.ajax
+ * @param {number} retryCount - Current retry count (internal use)
+ * @returns {Promise} - A promise that resolves or rejects based on the ajax call result
+ */
+function ajaxWithRetry(options, retryCount = 0) {
+    const isDatasetApi = options.url.includes('/api/datasets');
+
+    const runAjax = () => {
+        const internalOptions = { ...options };
+        // We handle callbacks manually to control retries
+        delete internalOptions.success;
+        delete internalOptions.error;
+
+        return $.ajax(internalOptions).then(
+            function(data, textStatus, jqXHR) {
+                if (options.success) {
+                    options.success.call(options.context || this, data, textStatus, jqXHR);
+                }
+                return data;
+            },
+            function(jqXHR, textStatus, errorThrown) {
+                const status = jqXHR.status;
+
+                if (isDatasetApi) {
+                    // 429 retry logic for dataset API (but not 50x)
+                    if (status === 429 && retryCount < uploadUrlMaxRetries && directUploadEnabled) {
+                        let retryAfterDelayMs = getRetryAfterDelayMs(jqXHR);
+
+                        // Recovery wait time for this specific 429 to clear
+                        let recoveryDelayMs = Math.max(
+                            retryAfterDelayMs || 0,
+                            Math.min(uploadUrlBaseRetryDelayMs * Math.pow(2, retryCount), uploadUrlMaxRetryDelayMs)
+                        );
+
+                        // Increase the persistent inter-request delay to slow down future calls
+                        uploadUrlInterRequestDelayMs += 50;
+                        updateUploadUrlCooldown(recoveryDelayMs);
+
+                        console.log(`Retrying call to ${options.url} due to 429 in ${recoveryDelayMs}ms (attempt ${retryCount + 1} of ${uploadUrlMaxRetries})`);
+
+                        return sleep(recoveryDelayMs).then(() => ajaxWithRetry(options, retryCount + 1));
+                    }
+                } else {
+                    // 50x retry logic for S3 (but not 429s)
+                    if (status >= 500 && status <= 599 && retryCount < 3) {
+                        const baseDelay = 100;
+                        const delay = retryCount === 0 ? baseDelay : baseDelay * Math.pow(2, retryCount);
+                        console.log(`Retrying call to ${options.url} due to ${status} in ${delay}ms (attempt ${retryCount + 1} of 3)`);
+                        return sleep(delay).then(() => ajaxWithRetry(options, retryCount + 1));
+                    }
+                }
+
+                if (options.error) {
+                    options.error.call(options.context || this, jqXHR, textStatus, errorThrown);
+                }
+                // Propagate the error. Using Deferred to mimic jqXHR rejection if needed
+                return $.Deferred().rejectWith(options.context || this, [jqXHR, textStatus, errorThrown]);
+            }
+        );
+    };
+
+    if (isDatasetApi) {
+        return waitForUploadUrlCooldown().then(() => {
+            recordUploadUrlRequest();
+            return runAjax();
+        });
+    } else {
+        return runAjax();
+    }
+}
+
 var fileUpload = class fileUploadClass {
     constructor(file) {
         this.file = file;
@@ -872,11 +946,8 @@ var fileUpload = class fileUploadClass {
         this.requestDirectUploadUrls();
     }
 
-    async requestDirectUploadUrls(retryCount = 0) {
-        await waitForUploadUrlCooldown();
-        recordUploadUrlRequest();
-
-        $.ajax({
+    requestDirectUploadUrls() {
+        ajaxWithRetry({
             url: siteUrl + '/api/datasets/:persistentId/uploadurls?persistentId=' + datasetPid + '&size=' + this.file.size,
             headers: { "X-Dataverse-key": apiKey },
             type: 'GET',
@@ -895,37 +966,10 @@ var fileUpload = class fileUploadClass {
                 this.doUpload();
                 console.log(JSON.stringify(data));
             },
-                            error: async function(jqXHR, textStatus, errorThrown) {
+            error: function(jqXHR, textStatus, errorThrown) {
                 console.log('Failure: ' + jqXHR.status);
                 console.log('Failure: ' + errorThrown);
 
-                if (jqXHR.status === 429 && retryCount < uploadUrlMaxRetries && directUploadEnabled) {
-                    let retryAfterDelayMs = getRetryAfterDelayMs(jqXHR);
-
-                    // Recovery wait time for this specific 429 to clear
-                    let recoveryDelayMs = Math.max(
-                        retryAfterDelayMs || 0,
-                        Math.min(uploadUrlBaseRetryDelayMs * Math.pow(2, retryCount), uploadUrlMaxRetryDelayMs)
-                    );
-
-                    // Increase the persistent inter-request delay to slow down future calls
-                    // Adding 50ms to the delay each time a 429 is encountered
-                    uploadUrlInterRequestDelayMs +=50;
-
-                    updateUploadUrlCooldown(recoveryDelayMs);
-
-                    console.log(
-                        'Received 429 while requesting upload URL for ' + this.file.name +
-                        '. Recovery wait: ' + recoveryDelayMs + 'ms. Persistent delay increased to: ' +
-                        uploadUrlInterRequestDelayMs + 'ms. Attempt ' +
-                        (retryCount + 1) + ' of ' + uploadUrlMaxRetries
-                    );
-
-                    await sleep(recoveryDelayMs);
-                    this.requestDirectUploadUrls(retryCount + 1);
-                    return;
-                }
-
                 inDataverseCall = false;
                 // If it hasn't been moved to processing yet, do it now so the count is right
                 if (fileList.indexOf(this) !== -1) {
@@ -983,14 +1027,15 @@ var fileUpload = class fileUploadClass {
             const uploadHeaders = this.urls.url.toLowerCase().includes("x-amz-tagging")
                 ? { "x-amz-tagging": "dv-state=temp" }
                 : {};
-            $.ajax({
+            ajaxWithRetry({
                 url: this.urls.url,
                 headers: uploadHeaders,
                 type: 'PUT',
                 data: this.file,
                 context: this,
                 cache: false,
                 processData: false,
+                contentType: false,
                 success: function() {
                     //ToDo - cancelling abandons the file. It is marked as temp so can be cleaned up later, but would be good to remove now (requires either sending a presigned delete URL or adding a callback to delete only a temp file
                     if (!cancelled) {
@@ -1053,14 +1098,15 @@ var fileUpload = class fileUploadClass {
                     const uploadHeaders = value.toLowerCase().includes("x-amz-tagging")
                         ? { "x-amz-tagging": "dv-state=temp" }
                         : {};
-                    $.ajax({
+                    ajaxWithRetry({
                         url: value,
                         headers: uploadHeaders,
                         type: 'PUT',
                         data: blob,
                         context: this,
                         cache: false,
                         processData: false,
+                        contentType: false,
                         success: function(data, status, response) {
                             console.log('Successful upload of part ' + key + ' of ' + Object.keys(this.urls.urls).length);
                             //The header has quotes around the eTag
@@ -1160,7 +1206,7 @@ var fileUpload = class fileUploadClass {
         }
     }
     async cancelMPUpload() {
-        $.ajax({
+        ajaxWithRetry({
             url: siteUrl + this.urls.abort,
             headers: { "X-Dataverse-key": apiKey },
             type: 'DELETE',
@@ -1181,7 +1227,7 @@ var fileUpload = class fileUploadClass {
         for (var i = 1; i <= this.numEtags; i++) {
             eTagsObject[i] = this.etags[i];
         }
-        $.ajax({
+        ajaxWithRetry({
             url: siteUrl + this.urls.complete,
             type: 'PUT',
             headers: { "X-Dataverse-key": apiKey },
@@ -1562,7 +1608,7 @@ async function directUploadFinished() {
                 // Remove the refresh button when uploads start
                 removeRefreshButton();
 
-                $.ajax({
+                ajaxWithRetry({
                     url: siteUrl + '/api/datasets/:persistentId/addFiles?persistentId=' + datasetPid,
                     headers: { "X-Dataverse-key": apiKey },
                     type: 'POST',