weko#61752 added s3 readonly access key

modules/invenio-files-rest/invenio_files_rest/admin.py

@@ -95,6 +95,8 @@ class LocationModelView(ModelView):
         quota_size=_('Quota Size'),
         access_key=_('Access Key'),
         secret_key=_('Secret Key'),
+        readonly_access_key=_('Readonly Access Key'),
+        readonly_secret_key=_('Readonly Secret Key'),
         s3_endpoint_url=_('S3_ENDPOINT_URL'),
         s3_send_file_directly=_('S3_SEND_FILE_DIRECTLY'),
         s3_default_block_size=_('S3_DEFAULT_BLOCK_SIZE'),
@@ -108,7 +110,7 @@ class LocationModelView(ModelView):
     column_default_sort = 'name'
     form_base_class = SecureForm
     form_columns = (
-        'name', 'uri', 'type', 'access_key', 'secret_key',
+        'name', 'uri', 'type', 'access_key', 'secret_key', 'readonly_access_key', 'readonly_secret_key',
         's3_endpoint_url', 's3_send_file_directly',
         's3_default_block_size', 's3_maximum_number_of_parts',
         's3_region_name', 's3_signature_version', 's3_url_expiration',
@@ -117,11 +119,18 @@ class LocationModelView(ModelView):
         'type': LazyChoices(
             lambda: current_app.config['FILES_REST_LOCATION_TYPE_LIST'])
     }
+    form_widget_args = {
+        'type': {'onchange': 'checkLocationType()'}
+    }
     form_extra_fields = {
         'access_key': PasswordField('access_key',
                                     widget=PasswordInput(hide_value=False)),
         'secret_key': PasswordField('secret_key',
                                     widget=PasswordInput(hide_value=False)),
+        'readonly_access_key': PasswordField('readonly_access_key',
+                                             widget=PasswordInput(hide_value=False)),
+        'readonly_secret_key': PasswordField('readonly_secret_key',
+                                             widget=PasswordInput(hide_value=False)),
         's3_endpoint_url': StringField('endpoint_url'),
         's3_send_file_directly': BooleanField('send_file_directly'),
         's3_default_block_size': IntegerField('default_block_size', validators=[NumberRange(min=0), Optional()]),

modules/invenio-files-rest/invenio_files_rest/models.py

@@ -288,6 +288,10 @@ class Location(db.Model, Timestamp):
 
     secret_key = db.Column(db.String(128), nullable=True)
 
+    readonly_access_key = db.Column(db.String(128), nullable=True)
+
+    readonly_secret_key = db.Column(db.String(128), nullable=True)
+
     s3_endpoint_url = db.Column(db.String(128), nullable=True)
 
     s3_send_file_directly = db.Column(db.Boolean(name='s3_send_file_directly'), nullable=False, default=True)

modules/invenio-files-rest/invenio_files_rest/storage/pyfs.py

@@ -45,7 +45,7 @@ def __init__(self, fileurl, size=None, modified=None, clean_dir=True, location=N
         self.location = location
         super(PyFSFileStorage, self).__init__(size=size, modified=modified)
 
-    def _get_fs(self, create_dir=True):
+    def _get_fs(self, create_dir=True, mode='rb'):
         """Return tuple with filesystem and filename."""
         filedir = dirname(self.fileurl)
         filename = basename(self.fileurl)
@@ -60,7 +60,7 @@ def open(self, mode='rb'):
 
         The caller is responsible for closing the file.
         """
-        fs, path = self._get_fs()
+        fs, path = self._get_fs(mode=mode)
         return fs.open(path, mode=mode)
 
     def delete(self):
@@ -69,7 +69,7 @@ def delete(self):
         The base directory is also removed, as it is assumed that only one file
         exists in the directory.
         """
-        fs, path = self._get_fs(create_dir=False)
+        fs, path = self._get_fs(create_dir=False, mode='wb')
         if fs.exists(path):
             fs.remove(path)
         if self.clean_dir and fs.exists('.'):
@@ -78,7 +78,7 @@ def delete(self):
 
     def initialize(self, size=0):
         """Initialize file on storage and truncate to given size."""
-        fs, path = self._get_fs()
+        fs, path = self._get_fs(mode='wb')
 
         # Required for reliably opening the file on certain file systems:
         if fs.exists(path):

modules/invenio-files-rest/invenio_files_rest/templates/admin/location_edit.html

@@ -6,12 +6,16 @@
   return document.evaluate(path, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
 }
 
-window.onload = function(){
+function checkLocationType() {
   var typeSpan = getElementByXpath("//label[@for='type']/..//span[@class='select2-selection__rendered']")
   var accessKeyField = getElementByXpath("//label[@for='access_key']/..")
   var accessKeyInput = getElementByXpath("//label[@for='access_key']/..//input")
   var secretKeyField = getElementByXpath("//label[@for='secret_key']/..")
   var secretKeyInput = getElementByXpath("//label[@for='secret_key']/..//input")
+  var readonlyAccessKeyField = getElementByXpath("//label[@for='readonly_access_key']/..")
+  var readonlyAccessKeyInput = getElementByXpath("//label[@for='readonly_access_key']/..//input")
+  var readonlySecretKeyField = getElementByXpath("//label[@for='readonly_secret_key']/..")
+  var readonlySecretKeyInput = getElementByXpath("//label[@for='readonly_secret_key']/..//input")
   var s3EndpointUrlField = getElementByXpath("//label[@for='s3_endpoint_url']/..")
   var s3EndpointUrlInput = getElementByXpath("//label[@for='s3_endpoint_url']/..//input")
   var s3SendFileDirectlyField = getElementByXpath("//label[@for='s3_send_file_directly']/..")
@@ -26,77 +30,105 @@
   var s3SignatureVersionInput = getElementByXpath("//label[@for='s3_signature_version']/..//input")
   var s3UrlExpiration = getElementByXpath("//label[@for='s3_url_expiration']/..")
   var s3UrlExpirationInput = getElementByXpath("//label[@for='s3_url_expiration']/..//input")
-
-  function checkLocationType() {
-    const labels = document.querySelectorAll('label');
-    let content = '';
-    content = s3MaximumNumberOfParts.textContent;
-    labels.forEach(label => {
-                const forId = label.getAttribute('for');
-                if (forId === "s3_maximum_number_of_parts") {
-                  content = label.textContent;
-                  if (content.length > 8) {
-                    content = content.slice(0, 8) + '\n' + content.slice(8);
-                    label.textContent = content;
-                  }
+  const labels = document.querySelectorAll('label');
+  let content = '';
+  content = s3MaximumNumberOfParts.textContent;
+  labels.forEach(label => {
+              const forId = label.getAttribute('for');
+              if (forId === "s3_maximum_number_of_parts") {
+                content = label.textContent;
+                if (content.length > 8) {
+                  content = content.slice(0, 8) + '\n' + content.slice(8);
+                  label.textContent = content;
                 }
-            });
-    if (typeSpan.getAttribute('title') == 'S3 Path') {
-      accessKeyField.style.display = ''
-      accessKeyInput.required = true
-      secretKeyField.style.display = ''
-      secretKeyInput.required = true
-      s3EndpointUrlField.style.display = ''
-      s3SendFileDirectlyField.style.display = ''
-      if (s3DefaultBlockSizeInput.value === ''){
-        s3DefaultBlockSizeInput.value = 5242880
-      }
-      if (s3MaximumNumberOfPartsInput.value === ''){
-        s3MaximumNumberOfPartsInput.value = 10000
-      }
-      if (s3UrlExpirationInput.value === ''){
-        s3UrlExpirationInput.value = 60
-      }
-      s3RegionNameInput.required = true
-    } else if (typeSpan.getAttribute('title') == 'S3 Virtural Host') {
-      accessKeyField.style.display = ''
-      accessKeyInput.required = true
-      secretKeyField.style.display = ''
-      secretKeyInput.required = true
-      s3EndpointUrlField.style.display = 'none'
-      s3SendFileDirectlyField.style.display = ''
-      if (s3DefaultBlockSizeInput.value === ''){
-        s3DefaultBlockSizeInput.value = 5242880
-      }
-      if (s3MaximumNumberOfPartsInput.value === ''){
-        s3MaximumNumberOfPartsInput.value = 10000
-      }
-      if (s3UrlExpirationInput.value === ''){
-        s3UrlExpirationInput.value = 60
-      }
-      s3RegionNameInput.required = true
-    } else {
-      accessKeyField.style.display = 'none'
-      secretKeyField.style.display = 'none'
-      s3EndpointUrlField.style.display = 'none'
-      s3SendFileDirectlyField.style.display = 'none'
-      s3DefaultBlockSize.style.display = 'none'
-      s3MaximumNumberOfParts.style.display = 'none'
-      s3RegionName.style.display = 'none'
-      s3SignatureVersion.style.display = 'none'
-      s3UrlExpiration.style.display = 'none'
-      accessKeyInput.value = null
-      accessKeyInput.required = false
-      secretKeyInput.value = null
-      secretKeyInput.required = false
-      s3EndpointUrlInput.value = null
-      s3DefaultBlockSizeInput.value = null
-      s3MaximumNumberOfPartsInput.value = null
-      s3RegionNameInput.value = null
-      s3RegionNameInput.required = false
-      s3UrlExpirationInput.value = null
+              }
+          });
+  if (typeSpan.getAttribute('title') == 'S3 Path') {
+    accessKeyField.style.display = ''
+    accessKeyInput.required = true
+    secretKeyField.style.display = ''
+    secretKeyInput.required = true
+    readonlyAccessKeyField.style.display = ''
+    readonlyAccessKeyInput.required = true
+    readonlySecretKeyField.style.display = ''
+    readonlySecretKeyInput.required = true
+    s3EndpointUrlField.style.display = ''
+    s3SendFileDirectlyField.style.display = ''
+    s3DefaultBlockSize.style.display = ''
+    if (s3DefaultBlockSizeInput.value === ''){
+      s3DefaultBlockSizeInput.value = 5242880
+    }
+    s3MaximumNumberOfParts.style.display = ''
+    if (s3MaximumNumberOfPartsInput.value === ''){
+      s3MaximumNumberOfPartsInput.value = 10000
+    }
+    s3UrlExpiration.style.display = ''
+    if (s3UrlExpirationInput.value === ''){
+      s3UrlExpirationInput.value = 60
+    }
+    s3RegionName.style.display = ''
+    s3RegionNameInput.required = true
+    s3SignatureVersion.style.display = ''
+  } else if (typeSpan.getAttribute('title') == 'S3 Virtural Host') {
+    accessKeyField.style.display = ''
+    accessKeyInput.required = true
+    secretKeyField.style.display = ''
+    secretKeyInput.required = true
+    readonlyAccessKeyField.style.display = ''
+    readonlyAccessKeyInput.required = true
+    readonlySecretKeyField.style.display = ''
+    readonlySecretKeyInput.required = true
+    s3EndpointUrlField.style.display = 'none'
+    s3SendFileDirectlyField.style.display = ''
+    s3DefaultBlockSize.style.display = ''
+    if (s3DefaultBlockSizeInput.value === ''){
+      s3DefaultBlockSizeInput.value = 5242880
     }
+    s3MaximumNumberOfParts.style.display = ''
+    if (s3MaximumNumberOfPartsInput.value === ''){
+      s3MaximumNumberOfPartsInput.value = 10000
+    }
+    s3UrlExpiration.style.display = ''
+    if (s3UrlExpirationInput.value === ''){
+      s3UrlExpirationInput.value = 60
+    }
+    s3RegionName.style.display = ''
+    s3RegionNameInput.required = true
+    s3SignatureVersion.style.display = ''
+  } else {
+    accessKeyField.style.display = 'none'
+    secretKeyField.style.display = 'none'
+    readonlyAccessKeyField.style.display = 'none'
+    readonlySecretKeyField.style.display = 'none'
+    s3EndpointUrlField.style.display = 'none'
+    s3SendFileDirectlyField.style.display = 'none'
+    s3DefaultBlockSize.style.display = 'none'
+    s3MaximumNumberOfParts.style.display = 'none'
+    s3RegionName.style.display = 'none'
+    s3SignatureVersion.style.display = 'none'
+    s3UrlExpiration.style.display = 'none'
+    accessKeyInput.value = null
+    accessKeyInput.required = false
+    secretKeyInput.value = null
+    secretKeyInput.required = false
+    readonlyAccessKeyInput.value = null
+    readonlyAccessKeyInput.required = false
+    readonlySecretKeyInput.value = null
+    readonlySecretKeyInput.required = false
+    s3EndpointUrlInput.value = null
+    s3DefaultBlockSizeInput.value = null
+    s3MaximumNumberOfPartsInput.value = null
+    s3RegionNameInput.value = null
+    s3RegionNameInput.required = false
+    s3UrlExpirationInput.value = null
   }
+}
+window.onload = function(){
+  document.querySelector("label[for='access_key']").innerHTML += '<strong style="color: red;">*</strong>';
+  document.querySelector("label[for='secret_key']").innerHTML += '<strong style="color: red;">*</strong>';
+  document.querySelector("label[for='readonly_access_key']").innerHTML += '<strong style="color: red;">*</strong>';
+  document.querySelector("label[for='readonly_secret_key']").innerHTML += '<strong style="color: red;">*</strong>';
+  document.querySelector("label[for='s3_region_name']").innerHTML += '<strong style="color: red;">*</strong>';
   checkLocationType()
 }
 </script>

modules/invenio-s3/invenio_s3/config.py

@@ -49,6 +49,10 @@
 for more information.
 """
 
+S3_READONLY_ACCCESS_KEY_ID = None
+S3_READONLY_SECRECT_ACCESS_KEY = None
+"""The access key, secret key to use when downloading files from the client."""
+
 S3_URL_EXPIRATION = 60
 """Number of seconds the file serving URL will be valid.
 

modules/invenio-s3/invenio_s3/ext.py

@@ -27,7 +27,7 @@ def __init__(self, app=None):
             self.init_app(app)
 
     # @cached_property
-    def init_s3fs_info(self, location):
+    def init_s3fs_info(self, location, mode='wb'):
         """Gather all the information needed to start the S3FSFileSystem."""
         if 'S3_ACCCESS_KEY_ID' in current_app.config:
             current_app.config['S3_ACCESS_KEY_ID'] = current_app.config[
@@ -49,9 +49,20 @@ def init_s3fs_info(self, location):
                 DeprecationWarning
             )
 
+        if mode == 'wb':
+            access_key = location.access_key if location.access_key \
+                else current_app.config.get('S3_ACCESS_KEY_ID', '')
+            secret_key = location.secret_key if location.secret_key \
+                else current_app.config.get('S3_SECRET_ACCESS_KEY', '')
+        else:
+            access_key = location.readonly_access_key if location.readonly_access_key \
+                else current_app.config.get('S3_READONLY_ACCESS_KEY_ID', '')
+            secret_key = location.readonly_secret_key if location.readonly_secret_key \
+                else current_app.config.get('S3_READONLY_SECRET_ACCESS_KEY', '')
+
         info = dict(
-            key=current_app.config.get('S3_ACCESS_KEY_ID', ''),
-            secret=current_app.config.get('S3_SECRET_ACCESS_KEY', ''),
+            key=access_key,
+            secret=secret_key,
             client_kwargs={},
             config_kwargs={
                 's3': {
@@ -71,10 +82,10 @@ def init_s3fs_info(self, location):
         if region_name:
             info['client_kwargs']['region_name'] = region_name
 
-        if location.type == current_app.config.get('S3_LOCATION_TYPE_S3_PATH_VALUE') or \
-            location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE'):
-            info['key'] = location.access_key
-            info['secret'] = location.secret_key
+        if (location.type == current_app.config.get('S3_LOCATION_TYPE_S3_PATH_VALUE') or
+            location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE')):
+            info['key'] = access_key
+            info['secret'] = secret_key
             info['client_kwargs']['endpoint_url'] = location.s3_endpoint_url
             region_name = location.s3_region_name
             if region_name:

modules/invenio-s3/invenio_s3/storage.py

@@ -56,10 +56,10 @@ def __init__(self, fileurl, size, modified, clean_dir, location):
                 self.block_size = location.s3_default_block_size
         super(S3FSFileStorage, self).__init__(fileurl, size, modified, clean_dir, location)
 
-    def _get_fs(self, *args, **kwargs):
+    def _get_fs(self, mode='rb', *args, **kwargs):
         """Get PyFilesystem instance and S3 real path."""
         if self.location is None or self.location.type == None:
-            return super(S3FSFileStorage, self)._get_fs(*args, **kwargs)
+            return super(S3FSFileStorage, self)._get_fs(mode=mode, *args, **kwargs)
 
         url = self.fileurl
         if self.location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE'):
@@ -73,15 +73,15 @@ def _get_fs(self, *args, **kwargs):
                 sub_parts = parts[2].split('.')
                 url = 's3://' + sub_parts[0] + '/' + '/'.join(parts[3:])
 
-        info = current_app.extensions['invenio-s3'].init_s3fs_info(location=self.location)
+        info = current_app.extensions['invenio-s3'].init_s3fs_info(location=self.location, mode=mode)
         fs = s3fs.S3FileSystem(default_block_size=self.block_size, **info)
 
         return (fs, url)
 
     @set_blocksize
     def initialize(self, size=0):
         """Initialize file on storage and truncate to given size."""
-        fs, path = self._get_fs()
+        fs, path = self._get_fs(mode='wb')
 
         self.remove(fs, path)
         fp = fs.open(path, mode='wb')
@@ -116,7 +116,7 @@ def remove(self, fs, path):
 
     def delete(self):
         """Delete a file."""
-        fs, path = self._get_fs()
+        fs, path = self._get_fs(mode='wb')
         self.remove(fs, path)
         return True
 
@@ -215,7 +215,7 @@ def send_file(
             )
 
         try:
-            fs, path = self._get_fs()
+            fs, path = self._get_fs(mode='rb')
             s3_url_builder = partial(
                 fs.url, path, expires=current_app.config['S3_URL_EXPIRATION']
             )
@@ -245,7 +245,7 @@ def copy(self, src, *args, **kwargs):
         """
         if self.location:
             if self.location.type != None:
-                fs, path = self._get_fs()
+                fs, path = self._get_fs(mode='wb')
                 fs.copy(src.fileurl, path)
             else:
             # local repository