From 11341e965f4069b41b618a4fd80633470abe507f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Mari=C3=A9thoz?= Date: Tue, 20 Apr 2021 16:40:35 +0200 Subject: [PATCH] identifiers: add ARK identifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds command line interface to interact with a NMA ARK server. * Mints a new ARK identifier when a document is created. * Registers the ARK identifier at the document creation. * Marks the ARK persistent identifier as deleted when the document is removed. * Replace the persistent URL by the ARK resolver URL if it is relevant. * Closes #399. Co-Authored-by: Johnny Mariéthoz --- setup.py | 1 + sonar/config_sonar.py | 18 ++ sonar/modules/ark/__init__.py | 18 ++ sonar/modules/ark/api.py | 259 ++++++++++++++++++ sonar/modules/ark/cli.py | 128 +++++++++ sonar/modules/documents/api.py | 12 +- sonar/modules/documents/extensions.py | 69 +++++ .../documents/document-v1.0.0_src.json | 6 + .../v7/documents/document-v1.0.0.json | 3 + sonar/modules/documents/marshmallow/json.py | 1 + sonar/modules/documents/minters.py | 30 +- .../documents/templates/documents/record.html | 4 +- .../documents/test_documents_permissions.py | 20 +- tests/conftest.py | 23 ++ .../documents/cli/test_documents_cli_ark.py | 146 ++++++++++ tests/ui/documents/test_documents_views.py | 2 +- tests/utils.py | 92 +++++++ 17 files changed, 816 insertions(+), 16 deletions(-) create mode 100644 sonar/modules/ark/__init__.py create mode 100644 sonar/modules/ark/api.py create mode 100644 sonar/modules/ark/cli.py create mode 100644 sonar/modules/documents/extensions.py create mode 100644 tests/ui/documents/cli/test_documents_cli_ark.py diff --git a/setup.py b/setup.py index 3a159553c..537e9813f 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ 'flask.commands': [ 'fixtures = sonar.modules.cli:fixtures', 'documents = sonar.modules.documents.cli.documents:documents', + 'ark = sonar.modules.ark.cli:ark', 'oaiharvester = \ sonar.modules.documents.cli.oaiharvester:oaiharvester', 'utils = sonar.modules.cli:utils', diff --git a/sonar/config_sonar.py b/sonar/config_sonar.py index 03b73b6c3..40bcdc944 100644 --- a/sonar/config_sonar.py +++ b/sonar/config_sonar.py @@ -78,3 +78,21 @@ } } # Custom resources for organisations + +# ARK +# === + +# SONAR_ARK_USER = 'test' +"""Username for the NMA server.""" +# SONAR_ARK_PASSWORD = 'test' +"""Password for the NMA server.""" +# SONAR_ARK_RESOLVER = 'https://n2t.net' +"""ARK resolver URL.""" +# SONAR_ARK_NMA = 'https://www.arketype.ch' +"""ARK Name Mapping Authority: a service provider server.""" +# SONAR_ARK_NAAN = '99999' +"""ARK prefix corresponding to an organisation.""" +# SONAR_ARK_SCHEME = 'ark:' +"""ARK scheme.""" +# SONAR_ARK_SHOULDER = 'ffk3' +"""ARK Shoulder, can be multiple for a given organisation.""" diff --git a/sonar/modules/ark/__init__.py b/sonar/modules/ark/__init__.py new file mode 100644 index 000000000..980d5d63a --- /dev/null +++ b/sonar/modules/ark/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""ARK module.""" diff --git a/sonar/modules/ark/api.py b/sonar/modules/ark/api.py new file mode 100644 index 000000000..9213e7d12 --- /dev/null +++ b/sonar/modules/ark/api.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +"""ARK API.""" + +import requests +import xmltodict +from flask import current_app +from requests.auth import HTTPBasicAuth +from werkzeug.local import LocalProxy + + +class NMAServerError(Exception): + """Remote NMA server error.""" + + +class NMAUnauthorizedError(Exception): + """Bad credential for the remote NAM server.""" + + +def check_http_status(valid_status): + """Check the http status returned by the remote server. + + :param valid_status: A list of valid HTTP status. + :returns: The wrapper function, see python decorator. + """ + def decorator(func): + """Decorator. + + :param func: The input function to decorate. + :returns: The wrapper function, see python decorator. + """ + def wrapper(*args, **kwargs): + """This is the wrapper function for decorator. + + :param args: All the parameters as key value. + :param kwargs: All the parameters as dict. + :returns: The wrapper function, see python decorator. + """ + status_code, content = func(*args, **kwargs) + if status_code == 401: + raise NMAUnauthorizedError( + f'The server returns an unauthorized status,' + f' msg: {content}.' + ) + if status_code not in valid_status: + raise NMAServerError( # pragma: no cover + f'The server returns an invalid status('\ + f'{status_code}), msg: {content}') + if isinstance(content, str): + content = content.strip() + content = content.replace('success: ', '') + return content + return wrapper + return decorator + +class Ark: + """ARK Client for arketype.ch. + + More details: https://www.arketype.ch/doc/apidoc.html. + """ + + def __init__(self): + """Constructor.""" + self.init_config() + self._url_status = f'{self._nma}/status' + self._url_get = f'{self._nma}/id' + self._url_login = f'{self._nma}/login' + self._url_minter = \ + f'{self._nma}/shoulder/{self._scheme}/{self._naan}/{self._shoulder}' + self._url_resolve = self._resolver + + def config(self): + """String representation with config and urls.""" + return f""" +config: + user: {self._user} + password: {self._password} + resolver: {self._resolver} + nma: {self._nma} + naan: {self._naan} + scheme: {self._scheme} + shoulder: {self._shoulder} +urls: + get: {self._url_get} + minter: {self._url_minter} + status: {self._url_status} + login: {self._url_login} + resolve: {self._url_resolve} + +""" + + def init_config(self): + """Read the configuation from the current app.""" + config = current_app.config + for conf_key in config.keys(): + if conf_key.startswith('SONAR_ARK_'): + setattr(self, conf_key.replace('SONAR_ARK', '').lower(), config.get(conf_key)) + + def ark_from_id(self, _id): + """Translate an ARK from an id. + + :returns: an ARK identifier. + """ + return f'{self._scheme}/{self._naan}/{self._shoulder}{_id}' + + def resolver_url(self, _id): + """Translate an ARK from an id. + + :param _id: The record identifier. + :returns: The URL to resolve the given identifier. + """ + ark_id = self.ark_from_id(_id) + return f'{self._url_resolve}/{ark_id}' + + def target_url(self, pid, view='global'): + """Create an ARK target url from an record pid. + + :param pid: The record persistant identifier. + :param view: The organisiation view code. + """ + cfg = current_app.config + host_name = 'https://' + cfg.get('JSONSCHEMAS_HOST') + return '/'.join([host_name, view, 'documents', pid]) + + @check_http_status(valid_status=[200]) + def status(self): + """Get the ARK server status.""" + response = requests.get(self._url_status) + return response.status_code, response.text + + @check_http_status(valid_status=[200]) + def login(self): + """Test the credentials on the ARK server. + + :returns: A tuple of the HTTP status code and the text response. + """ + response = requests.get( + self._url_login, + auth=HTTPBasicAuth(self._user, self._password) + ) + return response.status_code, response.text + + @check_http_status(valid_status=[200, 400]) + def get(self, pid): + """Get the information given an identifier. + + :param pid: The record persistant identifier. + :returns: A tuple of the HTTP status code and the server response as + dict. + """ + url = f'{self._url_get}/{self._scheme}/{self._naan}/{self._shoulder}{pid}' + response = requests.get(url) + data = {} + if response.status_code != 400: + for line in response.text.split('\n'): + if line: + key, value = line.split(': ', 1) + if key in ['datacite']: + data[key] = xmltodict.parse(value.replace('%0A',' ')) + else: + data[key] = value + return response.status_code, data + + @check_http_status(valid_status=[302]) + def resolve(self, pid): + """Resolve an ARK and return the target. + + :param pid: The record persistant identifier. + :returns: A tuple of the HTTP status code and the target URL. + """ + url = self.resolver_url(pid) + response = requests.get(url, allow_redirects=False) + return response.status_code, response.headers.get('Location') + + @check_http_status(valid_status=[201]) + def create(self, pid, target, update_if_exists='yes'): + """Create a new ARK with a given id. + + :param pid: The record persistant identifier. + :param target: The ARK target URL. + :param update_if_exists: If True update instead of create. + :returns: A tuple of the HTTP status code and the text response. + """ + ark_id = self.ark_from_id(pid) + url = f'{self._url_get}/{ark_id}?update_if_exists={update_if_exists}' + response = requests.put( + url, + auth=HTTPBasicAuth(self._user, self._password), + data=f'_target: {target}' + ) + return response.status_code, response.text + + + @check_http_status(valid_status=[201]) + def mint(self, target): + """Generate and register a new ARK id. + + :param target: The ARK target URL. + :returns: A tuple of the HTTP status code and the text response. + """ + response = requests.post( + self._url_minter, + auth=HTTPBasicAuth(self._user, self._password), + data=f'_target: {target}') + return response.status_code, response.text + + @check_http_status(valid_status=[200]) + def update(self, pid, target): + """Update the given ARK. + + :param pid: The record persistant identifier. + :param target: The ARK target URL. + :returns: A tuple of the HTTP status code and the text response. + """ + ark_id = self.ark_from_id(pid) + url = f'{self._url_get}/{ark_id}' + response = requests.post( + url, + auth=HTTPBasicAuth(self._user, self._password), + data=f'_target: {target}\n_status: public' + ) + return response.status_code, response.text + + @check_http_status(valid_status=[200]) + def delete(self, pid): + """Mark an ARK as unavailable. + + :param pid: The record persistant identifier. + :returns: A tuple of the HTTP status code and the text response. + """ + ark_id = self.ark_from_id(pid) + url = f'{self._url_get}/{ark_id}' + response = requests.post( + url, + auth=HTTPBasicAuth(self._user, self._password), + data=f'_status: unavailable | removed' + ) + return response.status_code, response.text + + +# proxy on the ARK API if it is enable +current_ark = LocalProxy( + lambda: Ark() if current_app.config.get('SONAR_ARK_NMA') else None) diff --git a/sonar/modules/ark/cli.py b/sonar/modules/ark/cli.py new file mode 100644 index 000000000..e398bf409 --- /dev/null +++ b/sonar/modules/ark/cli.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""ARK CLI commands.""" + +import json + +import click +from flask.cli import with_appcontext + +from .api import NMAServerError, NMAUnauthorizedError, current_ark + + +@click.group() +@with_appcontext +def ark(): + """ARK utils commands.""" + if not current_ark: + click.secho('ARK is not enabled.', fg='red') + raise click.Abort() + +@ark.command() +@with_appcontext +def status(): + """Check the ARK NMA status.""" + try: + message = current_ark.status() + click.secho(f'message: {message}', fg='green') + except NMAServerError as e: # pragma: no cover + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +@click.argument('ark') +def get(ark): + """Get the ARK informations given an id.""" + try: + data = json.dumps(current_ark.get(ark), indent=2) + click.secho(f'{data}', fg='green') + except NMAServerError as e: # pragma: no cover + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +@click.argument('ark') +def resolve(ark): + """Resolve an ARK id.""" + try: + message = current_ark.resolve(ark) + click.secho(f'{message}', fg='green') + except NMAServerError as e: # pragma: no cover + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +def login(): + """Check NMA login.""" + try: + message = current_ark.login() + click.secho(f'{message}', fg='green') + except (NMAServerError, NMAUnauthorizedError) as e: + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +def config(): + """Dump the ARK client configurations.""" + click.secho(f'{current_ark.config()}') + +@ark.command() +@with_appcontext +@click.argument('target') +def mint(target): + """Generate and register a new ARK id.""" + try: + message = current_ark.mint(target) + click.secho(f'{message}', fg='green') + except (NMAServerError, NMAUnauthorizedError) as e: + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +@click.argument('identifier') +@click.argument('target') +def create(identifier, target): + """Create an new ARK with a given id.""" + try: + message = current_ark.create(identifier, target, update_if_exists='yes') + click.secho(f'{message}', fg='green') + except (NMAServerError, NMAUnauthorizedError) as e: + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +@click.argument('identifier') +@click.argument('target') +def update(identifier, target): + """Update the given ARK.""" + try: + message = current_ark.update(identifier, target) + click.secho(f'{message}', fg='green') + except (NMAServerError, NMAUnauthorizedError) as e: + click.secho(f'{e}', fg='red') + +@ark.command() +@with_appcontext +@click.argument('identifier') +def delete(identifier): + """Mark an ARK as unavailable.""" + try: + message = current_ark.delete(identifier) + click.secho(f'{message}', fg='green') + except (NMAServerError, NMAUnauthorizedError) as e: + click.secho(f'{e}', fg='red') diff --git a/sonar/modules/documents/api.py b/sonar/modules/documents/api.py index 0b4eb2870..77bfed76a 100644 --- a/sonar/modules/documents/api.py +++ b/sonar/modules/documents/api.py @@ -31,8 +31,10 @@ create_thumbnail_from_file from ..api import SonarIndexer, SonarRecord, SonarSearch +from ..ark.api import current_ark from ..fetchers import id_fetcher from ..providers import Provider +from .extensions import ArkDocumentExtension # provider DocumentProvider = type('DocumentProvider', (Provider, ), dict(pid_type='doc')) @@ -51,7 +53,6 @@ class Meta: index = 'documents' doc_types = [] - class DocumentRecord(SonarRecord): """Document record class.""" @@ -59,6 +60,7 @@ class DocumentRecord(SonarRecord): fetcher = document_pid_fetcher provider = DocumentProvider schema = 'documents/document-v1.0.0.json' + _extensions = [ArkDocumentExtension()] @staticmethod def get_permanent_link(host, pid, org=None): @@ -357,6 +359,14 @@ def is_open_access(self): return True + def get_ark_resolver_url(self): + """Get the ark resolver url. + + :returns: the URL to resolve the current identifier. + """ + if self.get('ark'): + return current_ark.resolver_url(self.get('pid')) + class DocumentIndexer(SonarIndexer): """Indexing documents in Elasticsearch.""" diff --git a/sonar/modules/documents/extensions.py b/sonar/modules/documents/extensions.py new file mode 100644 index 000000000..a742218d5 --- /dev/null +++ b/sonar/modules/documents/extensions.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""DocumentRecord Extensions.""" + +from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from invenio_records.extensions import RecordExtension + +from ..ark.api import current_ark + + +class ArkDocumentExtension(RecordExtension): + """Register/unregister Ark identifiers.""" + + def post_create(self, record): + """Called after a record is created. + + :param record: the invenio record instance to be processed. + """ + self._create_or_update_ark(record) + + def pre_commit(self, record): + """Called before a record is committed. + + :param record: the invenio record instance to be processed. + """ + self._create_or_update_ark(record) + + def post_delete(self, record, force=False): + """Called after a record is deleted. + + :param record: the invenio record instance to be processed. + :param force: unused. + """ + if record.get('ark'): + response = current_ark.delete(record.get('pid')) + ark_id = response.replace('success: ', '') + p = PersistentIdentifier.get('ark', ark_id) + p.delete() + + def _create_or_update_ark(self, record): + """Create or update the ARK identifier. + + :param record: the invenio record instance to be processed. + """ + if record.get('ark'): + org = record.replace_refs().get('organisation', [{}])[0] + pid = record.get('pid') + ark_id = current_ark.create( + pid, + current_ark.target_url(pid, org.get('code', 'global'))) + p = PersistentIdentifier.get('ark', ark_id) + if p.status == PIDStatus.RESERVED: + p.register() + diff --git a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json index 2f9322ecd..a96efd8e2 100644 --- a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json +++ b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0_src.json @@ -214,10 +214,16 @@ "type": "string", "minLength": 1 }, + "ark": { + "title": "ARK (Archival Resource Key)", + "type": "string", + "minLength": 1 + }, "organisation": { "title": "Organisations", "type": "array", "minItems": 1, + "maxItems": 1, "items": { "title": "Organisation", "type": "object", diff --git a/sonar/modules/documents/mappings/v7/documents/document-v1.0.0.json b/sonar/modules/documents/mappings/v7/documents/document-v1.0.0.json index c72203eda..2c2266cbd 100644 --- a/sonar/modules/documents/mappings/v7/documents/document-v1.0.0.json +++ b/sonar/modules/documents/mappings/v7/documents/document-v1.0.0.json @@ -90,6 +90,9 @@ "pid": { "type": "keyword" }, + "ark": { + "type": "keyword" + }, "organisation": { "type": "object", "properties": { diff --git a/sonar/modules/documents/marshmallow/json.py b/sonar/modules/documents/marshmallow/json.py index d30987519..4b8522f07 100644 --- a/sonar/modules/documents/marshmallow/json.py +++ b/sonar/modules/documents/marshmallow/json.py @@ -74,6 +74,7 @@ class DocumentMetadataSchemaV1(StrictKeysMixin): """Schema for the document metadata.""" pid = PersistentIdentifier() + ark = SanitizedUnicode() documentType = SanitizedUnicode() title = fields.List(fields.Dict()) partOf = fields.List(fields.Dict()) diff --git a/sonar/modules/documents/minters.py b/sonar/modules/documents/minters.py index 1c734838f..fb32b300a 100644 --- a/sonar/modules/documents/minters.py +++ b/sonar/modules/documents/minters.py @@ -22,9 +22,11 @@ from flask import current_app from invenio_oaiserver.minters import oaiid_minter from invenio_oaiserver.provider import OAIIDProvider -from invenio_pidstore.errors import PIDDoesNotExistError +from invenio_pidstore.errors import PIDAlreadyExists, PIDDoesNotExistError from invenio_pidstore.models import PersistentIdentifier, PIDStatus +from ..ark.api import current_ark + def id_minter(record_uuid, data, provider, pid_key='pid', object_type='rec'): """Document PID minter.""" @@ -45,13 +47,15 @@ def id_minter(record_uuid, data, provider, pid_key='pid', object_type='rec'): except PIDDoesNotExistError: oaiid_minter(record_uuid, data) - rerodoc_minter(record_uuid, data, pid_key) + external_minters(record_uuid, data, pid_key) return pid -def rerodoc_minter(record_uuid, data, pid_key='pid'): - """RERODOC minter. +def external_minters(record_uuid, data, pid_key='pid'): + """External minters. + + RERO DOC and ARK. :param record_uuid: Record UUID. :param data: Record data. @@ -67,8 +71,18 @@ def rerodoc_minter(record_uuid, data, pid_key='pid'): object_uuid=record_uuid, status=PIDStatus.REGISTERED) pid.redirect(PersistentIdentifier.get('doc', data[pid_key])) - return pid - except Exception: + except PIDAlreadyExists: pass - - return None + if not data.get('harvested') and current_ark: + ark_id = current_ark.ark_from_id(data[pid_key]) + try: + pid = PersistentIdentifier.create( + 'ark', + ark_id, + object_type='rec', + object_uuid=record_uuid, + status=PIDStatus.RESERVED) + # TODO: this minter is called twice why? + except PIDAlreadyExists: + pass + data['ark'] = ark_id diff --git a/sonar/modules/documents/templates/documents/record.html b/sonar/modules/documents/templates/documents/record.html index 81d51548c..7288981b9 100644 --- a/sonar/modules/documents/templates/documents/record.html +++ b/sonar/modules/documents/templates/documents/record.html @@ -281,9 +281,9 @@
{% endif %} - {% set link = record.get_permanent_link(request.host_url, record.pid, view_code) %} + {% set link = record.get_ark_resolver_url() if record.ark else record.get_permanent_link(request.host_url, record.pid, view_code) %}
- {{ _('Permalink') }} + {{ _('Persistent URL') }}
{{ link }} diff --git a/tests/api/documents/test_documents_permissions.py b/tests/api/documents/test_documents_permissions.py index a7f064fc0..65cd0bccc 100644 --- a/tests/api/documents/test_documents_permissions.py +++ b/tests/api/documents/test_documents_permissions.py @@ -21,6 +21,7 @@ from flask import url_for from invenio_accounts.testutils import login_user_via_session +from invenio_pidstore.models import PersistentIdentifier, PIDStatus def test_list(app, client, make_document, superuser, admin, moderator, @@ -85,7 +86,7 @@ def test_list(app, client, make_document, superuser, admin, moderator, def test_create(client, document_json, superuser, admin, moderator, submitter, - user): + user, mock_ark): """Test create documents permissions.""" headers = { 'Content-Type': 'application/json', @@ -138,11 +139,16 @@ def test_create(client, document_json, superuser, admin, moderator, submitter, assert res.status_code == 201 assert res.json['metadata']['organisation'][0][ '$ref'] == 'https://sonar.ch/api/organisations/org' + ark_id = res.json['metadata']['ark'] + assert ark_id.startswith('ark:/') + assert PersistentIdentifier.get('ark', ark_id).status == \ + PIDStatus.REGISTERED def test_read(client, document, make_user, superuser, admin, moderator, - submitter, user): + submitter, user, mock_ark): """Test read documents permissions.""" + # Not logged res = client.get( url_for('invenio_records_rest.doc_item', pid_value=document['pid'])) @@ -210,6 +216,7 @@ def test_read(client, document, make_user, superuser, admin, moderator, def test_update(client, document, make_user, superuser, admin, moderator, submitter, user): """Test update documents permissions.""" + headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' @@ -273,8 +280,9 @@ def test_update(client, document, make_user, superuser, admin, moderator, def test_delete(client, document, make_document, make_user, superuser, admin, - moderator, submitter, user): + moderator, submitter, user, mock_ark): """Test delete documents permissions.""" + # Not logged res = client.delete( url_for('invenio_records_rest.doc_item', pid_value=document['pid'])) @@ -306,6 +314,7 @@ def test_delete(client, document, make_document, make_user, superuser, admin, # Create a new document document = make_document() + ark_id = document['ark'] # Logged as admin of other organisation other_admin = make_user('admin', 'org2') @@ -316,6 +325,9 @@ def test_delete(client, document, make_document, make_user, superuser, admin, # Logged as superuser login_user_via_session(client, email=superuser['email']) + pid = document['pid'] res = client.delete( - url_for('invenio_records_rest.doc_item', pid_value=document['pid'])) + url_for('invenio_records_rest.doc_item', pid_value=pid)) assert res.status_code == 204 + assert PersistentIdentifier.get('ark', ark_id).status == \ + PIDStatus.DELETED diff --git a/tests/conftest.py b/tests/conftest.py index 27117b3ff..d3749dc38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,7 @@ from invenio_access.models import ActionUsers, Role from invenio_accounts.ext import hash_password from invenio_files_rest.models import Location +from utils import MockArkServer from sonar.modules.deposits.api import DepositRecord from sonar.modules.documents.api import DocumentRecord @@ -35,6 +36,19 @@ from sonar.proxies import sonar +@pytest.fixture(scope='function') +def mock_ark(app, monkeypatch): + """Mock for the ARK module.""" + # be sure that we do not make any request on the ARK server + monkeypatch.setattr('requests.get', + lambda *args, **kwargs: MockArkServer.get(*args, **kwargs)) + monkeypatch.setattr('requests.post', + lambda *args, **kwargs: MockArkServer.post(*args, **kwargs)) + monkeypatch.setattr('requests.put', + lambda *args, **kwargs: MockArkServer.put(*args, **kwargs)) + # enable ARK + monkeypatch.setitem(app.config, 'SONAR_ARK_NMA', 'https://www.arketype.ch') + @pytest.fixture(scope='module') def es(appctx): """Setup and teardown all registered Elasticsearch indices. @@ -87,6 +101,15 @@ def app_config(app_config): user_unique_id='id', ))) + # ARK + app_config['SONAR_ARK_USER'] = 'test' + app_config['SONAR_ARK_PASSWORD'] = 'test' + app_config['SONAR_ARK_RESOLVER'] = 'https://n2t.net' + # ARK is disabled by default + app_config['SONAR_ARK_NMA'] = None + app_config['SONAR_ARK_NAAN'] = '99999' + app_config['SONAR_ARK_SCHEME'] = 'ark:' + app_config['SONAR_ARK_SHOULDER'] = 'ffk3' return app_config diff --git a/tests/ui/documents/cli/test_documents_cli_ark.py b/tests/ui/documents/cli/test_documents_cli_ark.py new file mode 100644 index 000000000..1d9a23edf --- /dev/null +++ b/tests/ui/documents/cli/test_documents_cli_ark.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# +# Swiss Open Access Repository +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Test documents RERODOC cli commands.""" + +from click.testing import CliRunner + +from sonar.modules.ark import cli + + +def test_ark_disabled_cli(app, script_info): + """Test ARK command line interface with ARK disabled.""" + + runner = CliRunner() + + # server status + result = runner.invoke( + cli.ark, + ['config'], + obj=script_info) + assert 'ARK is not enabled.' in result.output + + +def test_ark_bad_auth_cli(app, script_info, monkeypatch, mock_ark): + """Test ARK command line interface with bad authentication.""" + + # enable ARK with bad credential + monkeypatch.setitem(app.config, 'SONAR_ARK_USER', 'unauthorized') + + runner = CliRunner() + + # server login + result = runner.invoke( + cli.ark, + ['login'], + obj=script_info) + assert 'unauthorized' in result.output + + # mint a new ark id + result = runner.invoke( + cli.mint, + ['https://sonar.ch/global/documents/1'], + obj=script_info) + assert 'unauthorized' in result.output + + # create a new ark id + result = runner.invoke( + cli.create, + ['1', 'https://sonar.ch/global/documents/1'], + obj=script_info) + assert 'unauthorized' in result.output + + # update the target of an existing ark id + result = runner.invoke( + cli.update, + ['1', 'https://sonar.ch/view1/documents/1'], + obj=script_info) + assert 'unauthorized' in result.output + + # invalidate a an existing ark id + result = runner.invoke( + cli.delete, + ['1'], + obj=script_info) + assert 'unauthorized' in result.output + + +def test_ark_cli(app, script_info, mock_ark): + """Test ARK command line interface.""" + + runner = CliRunner() + + # server status + result = runner.invoke( + cli.status, + obj=script_info) + assert 'message: EZID is up' in result.output + + # get ark information + # monkeypatch.setattr(Ark, 'get', ArkMock.get) + result = runner.invoke( + cli.get, + ['7'], + obj=script_info) + assert '"success": "ark:/99999/ffk37"' in result.output + + # ark resolution + result = runner.invoke( + cli.resolve, + ['7'], + obj=script_info) + assert result.output.startswith('http') + + # ark server login + result = runner.invoke( + cli.login, + obj=script_info) + assert 'session cookie returned' in result.output + + # ark server config + result = runner.invoke( + cli.config, + obj=script_info) + assert 'config' in result.output + + # mint a new ark id + result = runner.invoke( + cli.mint, + ['https://sonar.ch/global/documents/1'], + obj=script_info) + assert result.output.startswith('ark:/') + + # create a new ark id + result = runner.invoke( + cli.create, + ['1', 'https://sonar.ch/global/documents/1'], + obj=script_info) + assert result.output.startswith('ark:/') + + # update the target of an existing ark id + result = runner.invoke( + cli.update, + ['1', 'https://sonar.ch/view1/documents/1'], + obj=script_info) + assert result.output.startswith('ark:/') + + # invalidate a an existing ark id + result = runner.invoke( + cli.delete, + ['1'], + obj=script_info) + assert result.output.startswith('ark:/') diff --git a/tests/ui/documents/test_documents_views.py b/tests/ui/documents/test_documents_views.py index 35d2bec1c..05f14b07e 100644 --- a/tests/ui/documents/test_documents_views.py +++ b/tests/ui/documents/test_documents_views.py @@ -64,7 +64,7 @@ def test_search(app, client): resource_type='documents')).status_code == 404 -def test_detail(app, client, document_with_file): +def test_detail(app, client, document_with_file, mock_ark): """Test document detail page.""" assert client.get( url_for('invenio_records_ui.doc', view='global', diff --git a/tests/utils.py b/tests/utils.py index 2baa5b655..17e379e3d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -41,3 +41,95 @@ def mock_response(status=200, content="CONTENT", json_data=None, if json_data: mock_resp.json = mock.Mock(return_value=json_data) return mock_resp + +class MockArkServer: + """Mock an ARK Server.""" + + @staticmethod + def get(url, auth=None, *args, **kwargs): + """Mock the requests get function.""" + import requests + class Response: + """Dummy response object.""" + + status_code = 200 + text = '' + if auth and (auth.username != 'test' or auth.password != 'test'): + Response.status_code = 401 + Response.text = 'error: unauthorized' + return Response + # status + if url.startswith('https://www.arketype.ch/status'): + Response.text = 'success: EZID is up' + # get + elif url.startswith('https://www.arketype.ch/id/'): + ark_id = url.replace('https://www.arketype.ch/id/', '') + Response.text = f''' +success: {ark_id} +_updated: 1620802178 +_target: https://sonar.ch/global/documents/1 +_profile: erc +_export: yes +_owner: apitest +_ownergroup: apitest_g +_created: 1620802104 +_status: public +''' + # login + elif url.startswith('https://www.arketype.ch/login'): + Response.text = 'success: session cookie returned' + # resolve + elif url.startswith('https://n2t.net/'): + Response.status_code = 302 + Response.headers = dict( + Location='https://sonar.ch/global/documents/1') + else: + return requests.get(url, *args, **kwargs) + return Response + + @staticmethod + def put(url, auth=None, *args, **kwargs): + """Mock the requests put function.""" + class Response: + """Dummy response object.""" + + status_code = 200 + text = '' + if auth and (auth.username != 'test' or auth.password != 'test'): + Response.status_code = 401 + Response.text = 'error: unauthorized' + return Response + # create + if url.startswith('https://www.arketype.ch/id/ark:/99999/ffk3'): + Response.status_code = 201 + ark_id = url.replace('https://www.arketype.ch/id/', '')\ + .replace('?update_if_exists=yes', '') + Response.text = f'success: {ark_id}' + else: + return None + return Response + + @staticmethod + def post(url, auth=None, *args, **kwargs): + """Mock the requests post function.""" + + class Response: + """Dummy response object.""" + + status_code = 200 + text = '' + if auth and (auth.username != 'test' or auth.password != 'test'): + Response.status_code = 401 + Response.text = 'error: unauthorized' + return Response + # mint + if url.startswith('https://www.arketype.ch/shoulder/ark:/99999/ffk3'): + Response.status_code = 201 + Response.text = 'success: ark:/99999/ffk3xx' + # update + elif url.startswith('https://www.arketype.ch/id/ark:/99999/ffk3'): + ark_id = url.replace('https://www.arketype.ch/id/', '') + Response.text = f'success: {ark_id}' + else: + return None + return Response