diff --git a/modules/invenio-files-rest/invenio_files_rest/admin.py b/modules/invenio-files-rest/invenio_files_rest/admin.py index b0e6075c7a..a9d4b73731 100644 --- a/modules/invenio-files-rest/invenio_files_rest/admin.py +++ b/modules/invenio-files-rest/invenio_files_rest/admin.py @@ -95,6 +95,8 @@ class LocationModelView(ModelView): quota_size=_('Quota Size'), access_key=_('Access Key'), secret_key=_('Secret Key'), + readonly_access_key=_('Readonly Access Key'), + readonly_secret_key=_('Readonly Secret Key'), s3_endpoint_url=_('S3_ENDPOINT_URL'), s3_send_file_directly=_('S3_SEND_FILE_DIRECTLY'), s3_default_block_size=_('S3_DEFAULT_BLOCK_SIZE'), @@ -108,7 +110,7 @@ class LocationModelView(ModelView): column_default_sort = 'name' form_base_class = SecureForm form_columns = ( - 'name', 'uri', 'type', 'access_key', 'secret_key', + 'name', 'uri', 'type', 'access_key', 'secret_key', 'readonly_access_key', 'readonly_secret_key', 's3_endpoint_url', 's3_send_file_directly', 's3_default_block_size', 's3_maximum_number_of_parts', 's3_region_name', 's3_signature_version', 's3_url_expiration', @@ -117,11 +119,18 @@ class LocationModelView(ModelView): 'type': LazyChoices( lambda: current_app.config['FILES_REST_LOCATION_TYPE_LIST']) } + form_widget_args = { + 'type': {'onchange': 'checkLocationType()'} + } form_extra_fields = { 'access_key': PasswordField('access_key', widget=PasswordInput(hide_value=False)), 'secret_key': PasswordField('secret_key', widget=PasswordInput(hide_value=False)), + 'readonly_access_key': PasswordField('readonly_access_key', + widget=PasswordInput(hide_value=False)), + 'readonly_secret_key': PasswordField('readonly_secret_key', + widget=PasswordInput(hide_value=False)), 's3_endpoint_url': StringField('endpoint_url'), 's3_send_file_directly': BooleanField('send_file_directly'), 's3_default_block_size': IntegerField('default_block_size', validators=[NumberRange(min=0), Optional()]), diff --git a/modules/invenio-files-rest/invenio_files_rest/models.py b/modules/invenio-files-rest/invenio_files_rest/models.py index 7198b4cae8..bded562ed3 100644 --- a/modules/invenio-files-rest/invenio_files_rest/models.py +++ b/modules/invenio-files-rest/invenio_files_rest/models.py @@ -288,6 +288,10 @@ class Location(db.Model, Timestamp): secret_key = db.Column(db.String(128), nullable=True) + readonly_access_key = db.Column(db.String(128), nullable=True) + + readonly_secret_key = db.Column(db.String(128), nullable=True) + s3_endpoint_url = db.Column(db.String(128), nullable=True) s3_send_file_directly = db.Column(db.Boolean(name='s3_send_file_directly'), nullable=False, default=True) diff --git a/modules/invenio-files-rest/invenio_files_rest/storage/pyfs.py b/modules/invenio-files-rest/invenio_files_rest/storage/pyfs.py index 901f4835eb..db7014e807 100644 --- a/modules/invenio-files-rest/invenio_files_rest/storage/pyfs.py +++ b/modules/invenio-files-rest/invenio_files_rest/storage/pyfs.py @@ -45,7 +45,7 @@ def __init__(self, fileurl, size=None, modified=None, clean_dir=True, location=N self.location = location super(PyFSFileStorage, self).__init__(size=size, modified=modified) - def _get_fs(self, create_dir=True): + def _get_fs(self, create_dir=True, mode='rb'): """Return tuple with filesystem and filename.""" filedir = dirname(self.fileurl) filename = basename(self.fileurl) @@ -60,7 +60,7 @@ def open(self, mode='rb'): The caller is responsible for closing the file. """ - fs, path = self._get_fs() + fs, path = self._get_fs(mode=mode) return fs.open(path, mode=mode) def delete(self): @@ -69,7 +69,7 @@ def delete(self): The base directory is also removed, as it is assumed that only one file exists in the directory. """ - fs, path = self._get_fs(create_dir=False) + fs, path = self._get_fs(create_dir=False, mode='wb') if fs.exists(path): fs.remove(path) if self.clean_dir and fs.exists('.'): @@ -78,7 +78,7 @@ def delete(self): def initialize(self, size=0): """Initialize file on storage and truncate to given size.""" - fs, path = self._get_fs() + fs, path = self._get_fs(mode='wb') # Required for reliably opening the file on certain file systems: if fs.exists(path): diff --git a/modules/invenio-files-rest/invenio_files_rest/templates/admin/location_edit.html b/modules/invenio-files-rest/invenio_files_rest/templates/admin/location_edit.html index bc5f0d6002..a10e857a54 100644 --- a/modules/invenio-files-rest/invenio_files_rest/templates/admin/location_edit.html +++ b/modules/invenio-files-rest/invenio_files_rest/templates/admin/location_edit.html @@ -6,12 +6,16 @@ return document.evaluate(path, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue; } -window.onload = function(){ +function checkLocationType() { var typeSpan = getElementByXpath("//label[@for='type']/..//span[@class='select2-selection__rendered']") var accessKeyField = getElementByXpath("//label[@for='access_key']/..") var accessKeyInput = getElementByXpath("//label[@for='access_key']/..//input") var secretKeyField = getElementByXpath("//label[@for='secret_key']/..") var secretKeyInput = getElementByXpath("//label[@for='secret_key']/..//input") + var readonlyAccessKeyField = getElementByXpath("//label[@for='readonly_access_key']/..") + var readonlyAccessKeyInput = getElementByXpath("//label[@for='readonly_access_key']/..//input") + var readonlySecretKeyField = getElementByXpath("//label[@for='readonly_secret_key']/..") + var readonlySecretKeyInput = getElementByXpath("//label[@for='readonly_secret_key']/..//input") var s3EndpointUrlField = getElementByXpath("//label[@for='s3_endpoint_url']/..") var s3EndpointUrlInput = getElementByXpath("//label[@for='s3_endpoint_url']/..//input") var s3SendFileDirectlyField = getElementByXpath("//label[@for='s3_send_file_directly']/..") @@ -26,77 +30,105 @@ var s3SignatureVersionInput = getElementByXpath("//label[@for='s3_signature_version']/..//input") var s3UrlExpiration = getElementByXpath("//label[@for='s3_url_expiration']/..") var s3UrlExpirationInput = getElementByXpath("//label[@for='s3_url_expiration']/..//input") - - function checkLocationType() { - const labels = document.querySelectorAll('label'); - let content = ''; - content = s3MaximumNumberOfParts.textContent; - labels.forEach(label => { - const forId = label.getAttribute('for'); - if (forId === "s3_maximum_number_of_parts") { - content = label.textContent; - if (content.length > 8) { - content = content.slice(0, 8) + '\n' + content.slice(8); - label.textContent = content; - } + const labels = document.querySelectorAll('label'); + let content = ''; + content = s3MaximumNumberOfParts.textContent; + labels.forEach(label => { + const forId = label.getAttribute('for'); + if (forId === "s3_maximum_number_of_parts") { + content = label.textContent; + if (content.length > 8) { + content = content.slice(0, 8) + '\n' + content.slice(8); + label.textContent = content; } - }); - if (typeSpan.getAttribute('title') == 'S3 Path') { - accessKeyField.style.display = '' - accessKeyInput.required = true - secretKeyField.style.display = '' - secretKeyInput.required = true - s3EndpointUrlField.style.display = '' - s3SendFileDirectlyField.style.display = '' - if (s3DefaultBlockSizeInput.value === ''){ - s3DefaultBlockSizeInput.value = 5242880 - } - if (s3MaximumNumberOfPartsInput.value === ''){ - s3MaximumNumberOfPartsInput.value = 10000 - } - if (s3UrlExpirationInput.value === ''){ - s3UrlExpirationInput.value = 60 - } - s3RegionNameInput.required = true - } else if (typeSpan.getAttribute('title') == 'S3 Virtural Host') { - accessKeyField.style.display = '' - accessKeyInput.required = true - secretKeyField.style.display = '' - secretKeyInput.required = true - s3EndpointUrlField.style.display = 'none' - s3SendFileDirectlyField.style.display = '' - if (s3DefaultBlockSizeInput.value === ''){ - s3DefaultBlockSizeInput.value = 5242880 - } - if (s3MaximumNumberOfPartsInput.value === ''){ - s3MaximumNumberOfPartsInput.value = 10000 - } - if (s3UrlExpirationInput.value === ''){ - s3UrlExpirationInput.value = 60 - } - s3RegionNameInput.required = true - } else { - accessKeyField.style.display = 'none' - secretKeyField.style.display = 'none' - s3EndpointUrlField.style.display = 'none' - s3SendFileDirectlyField.style.display = 'none' - s3DefaultBlockSize.style.display = 'none' - s3MaximumNumberOfParts.style.display = 'none' - s3RegionName.style.display = 'none' - s3SignatureVersion.style.display = 'none' - s3UrlExpiration.style.display = 'none' - accessKeyInput.value = null - accessKeyInput.required = false - secretKeyInput.value = null - secretKeyInput.required = false - s3EndpointUrlInput.value = null - s3DefaultBlockSizeInput.value = null - s3MaximumNumberOfPartsInput.value = null - s3RegionNameInput.value = null - s3RegionNameInput.required = false - s3UrlExpirationInput.value = null + } + }); + if (typeSpan.getAttribute('title') == 'S3 Path') { + accessKeyField.style.display = '' + accessKeyInput.required = true + secretKeyField.style.display = '' + secretKeyInput.required = true + readonlyAccessKeyField.style.display = '' + readonlyAccessKeyInput.required = true + readonlySecretKeyField.style.display = '' + readonlySecretKeyInput.required = true + s3EndpointUrlField.style.display = '' + s3SendFileDirectlyField.style.display = '' + s3DefaultBlockSize.style.display = '' + if (s3DefaultBlockSizeInput.value === ''){ + s3DefaultBlockSizeInput.value = 5242880 + } + s3MaximumNumberOfParts.style.display = '' + if (s3MaximumNumberOfPartsInput.value === ''){ + s3MaximumNumberOfPartsInput.value = 10000 + } + s3UrlExpiration.style.display = '' + if (s3UrlExpirationInput.value === ''){ + s3UrlExpirationInput.value = 60 + } + s3RegionName.style.display = '' + s3RegionNameInput.required = true + s3SignatureVersion.style.display = '' + } else if (typeSpan.getAttribute('title') == 'S3 Virtural Host') { + accessKeyField.style.display = '' + accessKeyInput.required = true + secretKeyField.style.display = '' + secretKeyInput.required = true + readonlyAccessKeyField.style.display = '' + readonlyAccessKeyInput.required = true + readonlySecretKeyField.style.display = '' + readonlySecretKeyInput.required = true + s3EndpointUrlField.style.display = 'none' + s3SendFileDirectlyField.style.display = '' + s3DefaultBlockSize.style.display = '' + if (s3DefaultBlockSizeInput.value === ''){ + s3DefaultBlockSizeInput.value = 5242880 } + s3MaximumNumberOfParts.style.display = '' + if (s3MaximumNumberOfPartsInput.value === ''){ + s3MaximumNumberOfPartsInput.value = 10000 + } + s3UrlExpiration.style.display = '' + if (s3UrlExpirationInput.value === ''){ + s3UrlExpirationInput.value = 60 + } + s3RegionName.style.display = '' + s3RegionNameInput.required = true + s3SignatureVersion.style.display = '' + } else { + accessKeyField.style.display = 'none' + secretKeyField.style.display = 'none' + readonlyAccessKeyField.style.display = 'none' + readonlySecretKeyField.style.display = 'none' + s3EndpointUrlField.style.display = 'none' + s3SendFileDirectlyField.style.display = 'none' + s3DefaultBlockSize.style.display = 'none' + s3MaximumNumberOfParts.style.display = 'none' + s3RegionName.style.display = 'none' + s3SignatureVersion.style.display = 'none' + s3UrlExpiration.style.display = 'none' + accessKeyInput.value = null + accessKeyInput.required = false + secretKeyInput.value = null + secretKeyInput.required = false + readonlyAccessKeyInput.value = null + readonlyAccessKeyInput.required = false + readonlySecretKeyInput.value = null + readonlySecretKeyInput.required = false + s3EndpointUrlInput.value = null + s3DefaultBlockSizeInput.value = null + s3MaximumNumberOfPartsInput.value = null + s3RegionNameInput.value = null + s3RegionNameInput.required = false + s3UrlExpirationInput.value = null } +} +window.onload = function(){ + document.querySelector("label[for='access_key']").innerHTML += '*'; + document.querySelector("label[for='secret_key']").innerHTML += '*'; + document.querySelector("label[for='readonly_access_key']").innerHTML += '*'; + document.querySelector("label[for='readonly_secret_key']").innerHTML += '*'; + document.querySelector("label[for='s3_region_name']").innerHTML += '*'; checkLocationType() } diff --git a/modules/invenio-s3/invenio_s3/config.py b/modules/invenio-s3/invenio_s3/config.py index b158823c60..df9b9f084a 100644 --- a/modules/invenio-s3/invenio_s3/config.py +++ b/modules/invenio-s3/invenio_s3/config.py @@ -49,6 +49,10 @@ for more information. """ +S3_READONLY_ACCCESS_KEY_ID = None +S3_READONLY_SECRECT_ACCESS_KEY = None +"""The access key, secret key to use when downloading files from the client.""" + S3_URL_EXPIRATION = 60 """Number of seconds the file serving URL will be valid. diff --git a/modules/invenio-s3/invenio_s3/ext.py b/modules/invenio-s3/invenio_s3/ext.py index 96dc9e6f14..c72a412b32 100644 --- a/modules/invenio-s3/invenio_s3/ext.py +++ b/modules/invenio-s3/invenio_s3/ext.py @@ -27,7 +27,7 @@ def __init__(self, app=None): self.init_app(app) # @cached_property - def init_s3fs_info(self, location): + def init_s3fs_info(self, location, mode='wb'): """Gather all the information needed to start the S3FSFileSystem.""" if 'S3_ACCCESS_KEY_ID' in current_app.config: current_app.config['S3_ACCESS_KEY_ID'] = current_app.config[ @@ -49,9 +49,20 @@ def init_s3fs_info(self, location): DeprecationWarning ) + if mode == 'wb': + access_key = location.access_key if location.access_key \ + else current_app.config.get('S3_ACCESS_KEY_ID', '') + secret_key = location.secret_key if location.secret_key \ + else current_app.config.get('S3_SECRET_ACCESS_KEY', '') + else: + access_key = location.readonly_access_key if location.readonly_access_key \ + else current_app.config.get('S3_READONLY_ACCESS_KEY_ID', '') + secret_key = location.readonly_secret_key if location.readonly_secret_key \ + else current_app.config.get('S3_READONLY_SECRET_ACCESS_KEY', '') + info = dict( - key=current_app.config.get('S3_ACCESS_KEY_ID', ''), - secret=current_app.config.get('S3_SECRET_ACCESS_KEY', ''), + key=access_key, + secret=secret_key, client_kwargs={}, config_kwargs={ 's3': { @@ -71,10 +82,10 @@ def init_s3fs_info(self, location): if region_name: info['client_kwargs']['region_name'] = region_name - if location.type == current_app.config.get('S3_LOCATION_TYPE_S3_PATH_VALUE') or \ - location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE'): - info['key'] = location.access_key - info['secret'] = location.secret_key + if (location.type == current_app.config.get('S3_LOCATION_TYPE_S3_PATH_VALUE') or + location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE')): + info['key'] = access_key + info['secret'] = secret_key info['client_kwargs']['endpoint_url'] = location.s3_endpoint_url region_name = location.s3_region_name if region_name: diff --git a/modules/invenio-s3/invenio_s3/storage.py b/modules/invenio-s3/invenio_s3/storage.py index 53166ecc6d..cf3e09d5da 100644 --- a/modules/invenio-s3/invenio_s3/storage.py +++ b/modules/invenio-s3/invenio_s3/storage.py @@ -56,10 +56,10 @@ def __init__(self, fileurl, size, modified, clean_dir, location): self.block_size = location.s3_default_block_size super(S3FSFileStorage, self).__init__(fileurl, size, modified, clean_dir, location) - def _get_fs(self, *args, **kwargs): + def _get_fs(self, mode='rb', *args, **kwargs): """Get PyFilesystem instance and S3 real path.""" if self.location is None or self.location.type == None: - return super(S3FSFileStorage, self)._get_fs(*args, **kwargs) + return super(S3FSFileStorage, self)._get_fs(mode=mode, *args, **kwargs) url = self.fileurl if self.location.type == current_app.config.get('S3_LOCATION_TYPE_S3_VIRTUAL_HOST_VALUE'): @@ -73,7 +73,7 @@ def _get_fs(self, *args, **kwargs): sub_parts = parts[2].split('.') url = 's3://' + sub_parts[0] + '/' + '/'.join(parts[3:]) - info = current_app.extensions['invenio-s3'].init_s3fs_info(location=self.location) + info = current_app.extensions['invenio-s3'].init_s3fs_info(location=self.location, mode=mode) fs = s3fs.S3FileSystem(default_block_size=self.block_size, **info) return (fs, url) @@ -81,7 +81,7 @@ def _get_fs(self, *args, **kwargs): @set_blocksize def initialize(self, size=0): """Initialize file on storage and truncate to given size.""" - fs, path = self._get_fs() + fs, path = self._get_fs(mode='wb') self.remove(fs, path) fp = fs.open(path, mode='wb') @@ -116,7 +116,7 @@ def remove(self, fs, path): def delete(self): """Delete a file.""" - fs, path = self._get_fs() + fs, path = self._get_fs(mode='wb') self.remove(fs, path) return True @@ -215,7 +215,7 @@ def send_file( ) try: - fs, path = self._get_fs() + fs, path = self._get_fs(mode='rb') s3_url_builder = partial( fs.url, path, expires=current_app.config['S3_URL_EXPIRATION'] ) @@ -245,7 +245,7 @@ def copy(self, src, *args, **kwargs): """ if self.location: if self.location.type != None: - fs, path = self._get_fs() + fs, path = self._get_fs(mode='wb') fs.copy(src.fileurl, path) else: # local repository diff --git a/modules/invenio-s3/tests/conftest.py b/modules/invenio-s3/tests/conftest.py index 2778236e7f..268d50b2da 100644 --- a/modules/invenio-s3/tests/conftest.py +++ b/modules/invenio-s3/tests/conftest.py @@ -118,6 +118,8 @@ def location(location_path, database): type='s3', access_key='', secret_key='', + readonly_access_key='', + readonly_secret_key='', s3_endpoint_url="https://s3.amazonaws.com", s3_send_file_directly=True, s3_maximum_number_of_parts=1000, diff --git a/modules/invenio-s3/tests/test_ext.py b/modules/invenio-s3/tests/test_ext.py index fe644478e5..e607a8c622 100644 --- a/modules/invenio-s3/tests/test_ext.py +++ b/modules/invenio-s3/tests/test_ext.py @@ -25,9 +25,11 @@ def location(): name='testloc', uri='s3://testbucket', default=True, - type='s3_path', + type='s3', access_key='location-access-key', secret_key='location-secret-key', + readonly_access_key='location-readonly-access-key', + readonly_secret_key='location-readonly-secret-key', s3_endpoint_url='http://localhost:9000', s3_region_name='us-east-1', s3_signature_version='s3v4' @@ -39,13 +41,19 @@ def test_init_s3fs_info_with_global_config(app, location): with app.app_context(): invenio_s3 = InvenioS3(app) info = invenio_s3.init_s3fs_info(location) - assert info['key'] == 'location-access-key' assert info['secret'] == 'location-secret-key' assert info['client_kwargs']['endpoint_url'] == 'http://localhost:9000' assert info['client_kwargs']['region_name'] == 'us-east-1' assert info['config_kwargs']['signature_version'] == 's3v4' + info = invenio_s3.init_s3fs_info(location, mode='rb') + assert info['key'] == 'location-readonly-access-key' + assert info['secret'] == 'location-readonly-secret-key' + assert info['client_kwargs']['endpoint_url'] == 'http://localhost:9000' + assert info['client_kwargs']['region_name'] == 'us-east-1' + assert info['config_kwargs']['signature_version'] == 's3v4' + def test_init_s3fs_info_with_location_config(app, location): """Location設定を使用してS3FS情報を初期化するテスト。""" @@ -53,29 +61,45 @@ def test_init_s3fs_info_with_location_config(app, location): with app.app_context(): invenio_s3 = InvenioS3(app) info = invenio_s3.init_s3fs_info(location) - assert info['key'] == 'location-access-key' assert info['secret'] == 'location-secret-key' assert info['client_kwargs']['endpoint_url'] == 'http://localhost:9000' assert info['client_kwargs']['region_name'] == 'us-east-1' assert info['config_kwargs']['signature_version'] == 's3v4' + info = invenio_s3.init_s3fs_info(location, mode='rb') + assert info['key'] == 'location-readonly-access-key' + assert info['secret'] == 'location-readonly-secret-key' + assert info['client_kwargs']['endpoint_url'] == 'http://localhost:9000' + assert info['client_kwargs']['region_name'] == 'us-east-1' + assert info['config_kwargs']['signature_version'] == 's3v4' + def test_init_s3fs_info_with_typo_correction(app): """Typoがある設定キーを修正するテスト。""" app.config['S3_ACCCESS_KEY_ID'] = 'typo-access-key' app.config['S3_SECRECT_ACCESS_KEY'] = 'typo-secret-key' + app.config['S3_READONLY_ACCESS_KEY_ID'] = 'typo-readonly-access-key' + app.config['S3_READONLY_SECRET_ACCESS_KEY'] = 'typo-readonly-secret-key' with app.app_context(): invenio_s3 = InvenioS3(app) location = Location( name='testloc', uri='s3://testbucket', default=True, - type='s3_path', + type='s3', access_key='', secret_key='', + readonly_access_key='', + readonly_secret_key='', s3_endpoint_url='http://localhost:9000', s3_region_name='us-east-1', s3_signature_version='s3v4' ) info = invenio_s3.init_s3fs_info(location) + assert info['key'] == 'typo-access-key' + assert info['secret'] == 'typo-secret-key' + + info = invenio_s3.init_s3fs_info(location, mode='rb') + assert info['key'] == 'typo-readonly-access-key' + assert info['secret'] == 'typo-readonly-secret-key' diff --git a/modules/weko-search-ui/weko_search_ui/utils.py b/modules/weko-search-ui/weko_search_ui/utils.py index 7782a5e6d5..aaf5097516 100644 --- a/modules/weko-search-ui/weko_search_ui/utils.py +++ b/modules/weko-search-ui/weko_search_ui/utils.py @@ -2245,7 +2245,7 @@ def import_items_to_system( if list_unuse_uri: for uri in list_unuse_uri: file = current_files_rest.storage_factory(fileurl=uri, size=1) - fs, path = file._get_fs() + fs, path = file._get_fs(mode='wb') if fs.exists(path): file.delete() delete_cache_data(cache_key) diff --git a/postgresql/ddl/61660.sql b/postgresql/ddl/61660.sql new file mode 100644 index 0000000000..e96c37ba6e --- /dev/null +++ b/postgresql/ddl/61660.sql @@ -0,0 +1,2 @@ +ALTER TABLE files_location ADD COLUMN readonly_access_key CHARACTER VARYING(128); +ALTER TABLE files_location ADD COLUMN readonly_secret_key CHARACTER VARYING(128); \ No newline at end of file diff --git a/scripts/instance.cfg b/scripts/instance.cfg index 08dcb3d9cd..dd1a126c08 100644 --- a/scripts/instance.cfg +++ b/scripts/instance.cfg @@ -357,6 +357,8 @@ THEME_MATHJAX_CDN = '//cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.4/MathJax.js?c FILES_REST_STORAGE_FACTORY = 'invenio_s3.s3fs_storage_factory' S3_ACCCESS_KEY_ID = None S3_SECRET_ACCESS_KEY = None +S3_READONLY_ACCESS_KEY_ID = None +S3_READONLY_SECRET_ACCESS_KEY = None S3_SEND_FILE_DIRECTLY = True S3_ENDPOINT_URL = None