feat(s3): S3アドオンにSigV4署名を追加

Dockerfile

@@ -53,7 +53,6 @@ COPY ./addons/onedrive/requirements.txt /code/addons/onedrive/
 COPY ./addons/onedrivebusiness/requirements.txt /code/addons/onedrivebusiness/
 #COPY ./addons/osfstorage/requirements.txt ./addons/osfstorage/
 COPY ./addons/owncloud/requirements.txt ./addons/owncloud/
-COPY ./addons/s3/requirements.txt ./addons/s3/
 COPY ./addons/twofactor/requirements.txt ./addons/twofactor/
 #COPY ./addons/wiki/requirements.txt ./addons/wiki/
 COPY ./addons/zotero/requirements.txt ./addons/zotero/

addons/s3/README.md

@@ -16,3 +16,23 @@ If you already have an access key and ID, skip this step
 2. Scroll down to Configure Add-ons
 3. Connect your account and enter your ID and secret
 4. Select a bucket to work from, or create a new one.
+
+
+## Creating a restricted-access AWS user for S3 connection
+
+1. Login to AWS
+2. Open Identity and Access Management
+3. Create a user, assign name, set "Access Type" to "Programmatic Access"
+4. Permissions (simple):
+   1. Add "AmazonS3FullAccess" policy
+5. Permissions (minimal):
+   1. Click "Create Policy" => opens a new window
+   2. Make a new policy with the following Actions enabled: `["s3:DeleteObject", "s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:ReplicateObject", "s3:RestoreObject", "s3:ListAllMyBuckets", "s3:GetBucketLocation", "s3:CreateBucket"]`
+   3. Resources: *I just do "All Resources", but an AWS-educated person would know better about how to narrow it down.
+   4. Click "Create"
+   5. Return to "Create User" window
+   6. Reload policy list
+   7. Search for your new policy and select.
+6. Tags: *eh*
+7. Click "Review", review, then click "Create user"
+8. Note "Access key ID" and "Secret access key".  These are the inputs to the OSF S3 addon.

addons/s3/apps.py

@@ -12,6 +12,7 @@
 
 class S3AddonAppConfig(BaseAddonAppConfig):
 
+    default = True
     name = 'addons.s3'
     label = 'addons_s3'
     full_name = 'Amazon S3'

addons/s3/models.py

@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-
 from addons.base.models import (BaseOAuthNodeSettings, BaseOAuthUserSettings,
                                 BaseStorageAddon)
 from django.db import models
@@ -8,11 +6,18 @@
 from addons.base import exceptions
 from addons.s3.provider import S3Provider
 from addons.s3.serializer import S3Serializer
-from addons.s3.settings import (BUCKET_LOCATIONS,
-                                        ENCRYPT_UPLOADS_DEFAULT)
-from addons.s3.utils import (bucket_exists,
-                                     get_bucket_location_or_error,
-                                     get_bucket_names)
+from addons.s3.settings import (
+    BUCKET_LOCATIONS,
+    ENCRYPT_UPLOADS_DEFAULT
+)
+from addons.s3.utils import (
+    bucket_exists,
+    get_bucket_location_or_error,
+    get_bucket_names,
+    get_bucket_prefixes
+)
+from website.util import api_v2_url
+
 
 class S3FileNode(BaseFileNode):
     _provider = 's3'
@@ -53,10 +58,12 @@ def folder_path(self):
 
     @property
     def display_name(self):
-        return u'{0}: {1}'.format(self.config.full_name, self.folder_id)
+        return f'{self.config.full_name}: {self.folder_id}'
 
     def set_folder(self, folder_id, auth):
-        if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, folder_id):
+        bucket_name = folder_id.split(':')[0]
+
+        if not bucket_exists(self.external_account.oauth_key, self.external_account.oauth_secret, bucket_name):
             error_message = ('We are having trouble connecting to that bucket. '
                              'Try a different one.')
             raise exceptions.InvalidFolderError(error_message)
@@ -66,7 +73,7 @@ def set_folder(self, folder_id, auth):
         bucket_location = get_bucket_location_or_error(
             self.external_account.oauth_key,
             self.external_account.oauth_secret,
-            folder_id
+            bucket_name
         )
         try:
             bucket_location = BUCKET_LOCATIONS[bucket_location]
@@ -75,32 +82,46 @@ def set_folder(self, folder_id, auth):
             # Default to the key. When hit, add mapping to settings
             pass
 
-        self.folder_name = '{} ({})'.format(folder_id, bucket_location)
+        self.folder_name = f'{folder_id} ({bucket_location})'
         self.save()
 
-        self.nodelogger.log(action='bucket_linked', extra={'bucket': str(folder_id)}, save=True)
+        self.nodelogger.log(action='bucket_linked', extra={'bucket': bucket_name, 'path': self.folder_id}, save=True)
 
-    def get_folders(self, **kwargs):
-        # This really gets only buckets, not subfolders,
-        # as that's all we want to be linkable on a node.
-        try:
+    def get_folders(self, path, folder_id):
+        """
+        Our S3 implementation allows for folder_id to be a bucket's root, or a subfolder in that bucket.
+        """
+        # This is the root, so list all buckets.
+        if not folder_id:
             buckets = get_bucket_names(self)
-        except Exception:
-            raise exceptions.InvalidAuthError()
 
-        return [
-            {
+            return [{
                 'addon': 's3',
                 'kind': 'folder',
-                'id': bucket,
+                'id': f'{bucket}:/',
                 'name': bucket,
-                'path': bucket,
+                'bucket_name': bucket,
+                'path': '/',
                 'urls': {
-                    'folders': ''
+                    'folders': api_v2_url(
+                        f'nodes/{self.owner._id}/addons/s3/folders/',
+                        params={
+                            'id': bucket,
+                            'bucket_name': bucket,
+                        }
+                    ),
                 }
-            }
-            for bucket in buckets
-        ]
+            } for bucket in buckets]
+        # This is for a directory for a specific bucket, folders (Prefixes), but not files (Keys) returned, because
+        # these we can only set folders as our base folder_id
+        else:
+            bucket_name, _, path = folder_id.partition(':/')
+            return get_bucket_prefixes(
+                self.external_account.oauth_key,
+                self.external_account.oauth_secret,
+                prefix=path,
+                bucket_name=bucket_name
+            )
 
     @property
     def complete(self):
@@ -124,7 +145,7 @@ def deauthorize(self, auth=None, log=True):
 
     def delete(self, save=True):
         self.deauthorize(log=False)
-        super(NodeSettings, self).delete(save=save)
+        super().delete(save=save)
 
     def serialize_waterbutler_credentials(self):
         if not self.has_auth:
@@ -135,18 +156,24 @@ def serialize_waterbutler_credentials(self):
         }
 
     def serialize_waterbutler_settings(self):
+        """
+        We use the folder id to hold the bucket location
+        """
         if not self.folder_id:
             raise exceptions.AddonError('Cannot serialize settings for S3 addon')
+
+        bucket_name = self.folder_id.split(':')[0]
         return {
-            'bucket': self.folder_id,
+            'bucket': bucket_name,
+            'id': self.folder_id,
             'encrypt_uploads': self.encrypt_uploads
         }
 
     def create_waterbutler_log(self, auth, action, metadata):
         url = self.owner.web_url_for('addon_view_or_download_file', path=metadata['path'], provider='s3')
 
         self.owner.add_log(
-            's3_{0}'.format(action),
+            f's3_{action}',
             auth=auth,
             params={
                 'project': self.owner.parent_id,

addons/s3/provider.py

@@ -1,14 +1,14 @@
 from addons.s3.serializer import S3Serializer
 
-class S3Provider(object):
+class S3Provider:
     """An alternative to `ExternalProvider` not tied to OAuth"""
 
     name = 'Amazon S3'
     short_name = 's3'
     serializer = S3Serializer
 
     def __init__(self, account=None):
-        super(S3Provider, self).__init__()
+        super().__init__()
 
         # provide an unauthenticated session by default
         self.account = account

addons/s3/requirements.txt

@@ -1 +1,2 @@
-boto==2.38.0
+# boto (v2) was removed as part of SigV4 migration.
+# boto3/botocore are declared in the root requirements.txt.

addons/s3/settings/__init__.py

@@ -7,4 +7,4 @@
 try:
     from .local import *  # noqa
 except ImportError:
-    logger.warn('No local.py settings file found')
+    logger.warning('No local.py settings file found')

addons/s3/static/node-cfg.js

@@ -4,4 +4,4 @@ var s3NodeConfig = require('./s3NodeConfig.js').s3NodeConfig;
 
 var url = window.contextVars.node.urls.api + 's3/settings/';
 
-new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid');
+new s3NodeConfig('Amazon S3', '#s3Scope', url, '#s3Grid');

addons/s3/static/s3-rubeus-cfg.js

@@ -1,47 +1,46 @@
-    var Rubeus = require('rubeus');
+var Rubeus = require('rubeus');
 
-    Rubeus.cfg.s3 = {
+Rubeus.cfg.s3 = {
 
-        uploadMethod: 'PUT',
-        uploadUrl: null,
-        uploadAdded: function(file, item) {
-            var self = this;
-            var parent = self.getByID(item.parentID);
-            var name = file.name;
-            // Make it possible to upload into subfolders
-            while (parent.depth > 1 && !parent.isAddonRoot) {
-                name = parent.name + '/' + name;
-                parent = self.getByID(parent.parentID);
-            }
-            file.destination = name;
-            file.signedUrlFrom = parent.urls.upload;
-        },
+    uploadMethod: 'PUT',
+    uploadUrl: null,
+    uploadAdded: function (file, item) {
+        var self = this;
+        var parent = self.getByID(item.parentID);
+        var name = file.name;
+        // Make it possible to upload into subfolders
+        while (parent.depth > 1 && !parent.isAddonRoot) {
+            name = parent.name + '/' + name;
+            parent = self.getByID(parent.parentID);
+        }
+        file.destination = name;
+        file.signedUrlFrom = parent.urls.upload;
+    },
 
-        uploadSending: function(file, formData, xhr) {
-            xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream');
-            xhr.setRequestHeader('x-amz-acl', 'private');
-        },
+    uploadSending: function (file, formData, xhr) {
+        xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream');
+        xhr.setRequestHeader('x-amz-acl', 'private');
+    },
 
-        uploadSuccess: function(file, row) {
-            var self = this;
-            var parent = this.getByID(row.parentID);
-            row.urls = {
-                'delete': parent.nodeApiUrl + 's3/' + file.destination + '/',
-                'download': parent.nodeUrl + 's3/' + file.destination + '/download/',
-                'view': parent.nodeUrl + 's3/' + file.destination + '/'
-            };
-            row.permissions = parent.permissions;
-            this.updateItem(row);
-            var updated = Rubeus.Utils.itemUpdated(row, parent);
-            if (updated) {
-                self.changeStatus(row, Rubeus.Status.UPDATED);
-                self.delayRemoveRow(row);
-            } else {
-                self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000,
-                    function(row) {
-                        self.showButtons(row);
-                    });
-            }
+    uploadSuccess: function (file, row) {
+        var self = this;
+        var parent = this.getByID(row.parentID);
+        row.urls = {
+            'delete': parent.nodeApiUrl + 's3/' + file.destination + '/',
+            'download': parent.nodeUrl + 's3/' + file.destination + '/download/',
+            'view': parent.nodeUrl + 's3/' + file.destination + '/'
+        };
+        row.permissions = parent.permissions;
+        this.updateItem(row);
+        var updated = Rubeus.Utils.itemUpdated(row, parent);
+        if (updated) {
+            self.changeStatus(row, Rubeus.Status.UPDATED);
+            self.delayRemoveRow(row);
+        } else {
+            self.changeStatus(row, Rubeus.Status.UPLOAD_SUCCESS, null, 2000,
+                function (row) {
+                    self.showButtons(row);
+                });
         }
-    };
-
+    }
+};

addons/s3/static/s3AnonymousLogActionList.json

@@ -1,11 +1,11 @@
 {
-    "s3_bucket_linked" : "A user linked an Amazon S3 bucket to a project",
-    "s3_bucket_unlinked" : "A user unselected an Amazon S3 bucket in a project",
-    "s3_file_added" : "A user added a file to an Amazon S3 bucket in a project",
-    "s3_file_removed" : "A user removed a file in an Amazon S3 bucket in a project",
-    "s3_file_updated" : "A user updated a file in an Amazon S3 bucket in a project",
-    "s3_folder_created" : "A user created a folder in an Amazon S3 in a project",
-    "s3_node_authorized" : "A user authorized the Amazon S3 addon for a project",
-    "s3_node_deauthorized" : "A user deauthorized the Amazon S3 addon for a project",
-    "s3_node_deauthorized_no_user" : "Amazon S3 addon for a project deauthorized"
-}
+    "s3_bucket_linked": "A user linked an Amazon S3 bucket to a project",
+    "s3_bucket_unlinked": "A user unselected an Amazon S3 bucket in a project",
+    "s3_file_added": "A user added a file to an Amazon S3 bucket in a project",
+    "s3_file_removed": "A user removed a file in an Amazon S3 bucket in a project",
+    "s3_file_updated": "A user updated a file in an Amazon S3 bucket in a project",
+    "s3_folder_created": "A user created a folder in an Amazon S3 in a project",
+    "s3_node_authorized": "A user authorized the Amazon S3 addon for a project",
+    "s3_node_deauthorized": "A user deauthorized the Amazon S3 addon for a project",
+    "s3_node_deauthorized_no_user": "Amazon S3 addon for a project deauthorized"
+}

addons/s3/static/s3LogActionList.json

@@ -1,11 +1,11 @@
 {
-  "s3_bucket_linked" : "${user} linked the Amazon S3 bucket ${bucket} to ${node}",
-  "s3_bucket_unlinked" : "${user} unselected the Amazon S3 bucket ${bucket} in ${node}",
-  "s3_file_added" : "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}",
-  "s3_file_removed" : "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}",
-  "s3_file_updated" : "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}",
-  "s3_folder_created" : "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}",
-  "s3_node_authorized" : "${user} authorized the Amazon S3 addon for ${node}",
-  "s3_node_deauthorized" : "${user} deauthorized the Amazon S3 addon for ${node}",
-  "s3_node_deauthorized_no_user" : "Amazon S3 addon for ${node} deauthorized"
-}
+  "s3_bucket_linked": "${user} linked the Amazon S3 bucket ${bucket} to ${node}",
+  "s3_bucket_unlinked": "${user} unselected the Amazon S3 bucket ${bucket} in ${node}",
+  "s3_file_added": "${user} added file ${path} to Amazon S3 bucket ${bucket} in ${node}",
+  "s3_file_removed": "${user} removed ${path} in Amazon S3 bucket ${bucket} in ${node}",
+  "s3_file_updated": "${user} updated file ${path} in Amazon S3 bucket ${bucket} in ${node}",
+  "s3_folder_created": "${user} created folder ${path} in Amazon S3 bucket ${bucket} in ${node}",
+  "s3_node_authorized": "${user} authorized the Amazon S3 addon for ${node}",
+  "s3_node_deauthorized": "${user} deauthorized the Amazon S3 addon for ${node}",
+  "s3_node_deauthorized_no_user": "Amazon S3 addon for ${node} deauthorized"
+}