From 706f0a7acbf0ff91a33a1255463b61edf7b8ba87 Mon Sep 17 00:00:00 2001 From: Chiara Bigarella Date: Thu, 3 May 2018 16:00:52 +0200 Subject: [PATCH 1/6] models: don't override Type if the value is unkonwn --- asclepias_broker/models.py | 10 ++++++++++ tests/model/test_metadata.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/asclepias_broker/models.py b/asclepias_broker/models.py index c6ca3ec..77428ec 100644 --- a/asclepias_broker/models.py +++ b/asclepias_broker/models.py @@ -486,6 +486,8 @@ def update(self, payload, validate=True): new_json = deepcopy(self.json or {}) for k in OVERRIDABLE_KEYS: if payload.get(k): + if k == 'Type' and not _is_type_overridable(new_json, payload): + continue new_json[k] = payload[k] if validate: jsonschema.validate(new_json, self.SCHEMA) @@ -494,6 +496,14 @@ def update(self, payload, validate=True): return self +def _is_type_overridable(metadata, payload): + return not (metadata.get('Type') is not None and + metadata.get('Type') != {} and + metadata['Type'].get('Name') != 'unknown' and + (payload['Type'].get('Name') == 'unknown' or + payload['Type'].get('Name') == '')) + + class GroupRelationshipMetadata(db.Model, Timestamp): """Metadata for a group relationship.""" diff --git a/tests/model/test_metadata.py b/tests/model/test_metadata.py index a45bbbf..d8a7d48 100644 --- a/tests/model/test_metadata.py +++ b/tests/model/test_metadata.py @@ -81,6 +81,38 @@ def test_group_metadata(db): ) +def test_group_metadata_update_type(db): + g = Group(type=GroupType.Identity) + db.session.add(g) + db.session.commit() + gm = GroupMetadata(group_id=g.id) + db.session.add(gm) + db.session.commit() + assert g.data == gm + assert g.data.json == gm.json + + update_and_compare(gm, {'Title': 'Some title'}) + # Add Type + update_and_compare(gm, {'Title': 'Some other title', + 'Type': {'Name': 'unknown'}}) + + # Change Type + update_and_compare(gm, {'Title': 'Some other title', + 'Type': {'Name': 'software'}}) + + # Don't override Type + update_and_compare(gm, {'Type': {}}, {'Title': 'Some other title', + 'Type': {'Name': 'software'}}) + update_and_compare(gm, {'Type': {'Name': 'unknown'}}, + {'Title': 'Some other title', + 'Type': {'Name': 'software'}}) + + # Change Type + update_and_compare(gm, {'Type': {'Name': 'dataset'}}, + {'Title': 'Some other title', + 'Type': {'Name': 'dataset'}}) + + def test_group_relationship_metadata(db): g_src = Group(type=GroupType.Identity) g_trg = Group(type=GroupType.Identity) From e062cf0d3495cca20653bbb2eb11a1f8c8a4e977 Mon Sep 17 00:00:00 2001 From: Chiara Bigarella Date: Fri, 4 May 2018 13:54:53 +0200 Subject: [PATCH 2/6] indexer: fix build_doc --- asclepias_broker/indexer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/asclepias_broker/indexer.py b/asclepias_broker/indexer.py index b5b8d27..ea00d69 100644 --- a/asclepias_broker/indexer.py +++ b/asclepias_broker/indexer.py @@ -98,13 +98,11 @@ def index_documents(docs, bulk=False): def build_doc(rel, src_grp=None, trg_grp=None, grouping=None): """Build the ES document for a relationship.""" - if rel.type == GroupType.Identity: - # Fetch the supergroup (Version) of the Identity relations for metadata - src_meta = build_group_metadata(rel.source.supergroupsm2m[0].group) if src_grp: src_meta = build_group_metadata(src_grp) elif rel.type == GroupType.Identity: - pass + # Fetch the supergroup (Version) of the Identity relations for metadata + src_meta = build_group_metadata(rel.source.supergroupsm2m[0].group) else: src_meta = build_group_metadata(rel.source) From 8bb78e6d6e81028aeebdc911d0ced1bf5ed5ae61 Mon Sep 17 00:00:00 2001 From: Krzysztof Nowak Date: Fri, 4 May 2018 15:08:57 +0200 Subject: [PATCH 3/6] indexing: fixes an issue with documents being index twice Signed-off-by: Krzysztof Nowak --- asclepias_broker/indexer.py | 35 ++++++++++++++++++++++++----------- asclepias_broker/models.py | 6 ++++-- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/asclepias_broker/indexer.py b/asclepias_broker/indexer.py index ea00d69..337dad3 100644 --- a/asclepias_broker/indexer.py +++ b/asclepias_broker/indexer.py @@ -6,6 +6,7 @@ # under the terms of the MIT License; see LICENSE file for more details. """Elasticsearch indexing module.""" +import uuid from copy import deepcopy from itertools import chain @@ -98,24 +99,36 @@ def index_documents(docs, bulk=False): def build_doc(rel, src_grp=None, trg_grp=None, grouping=None): """Build the ES document for a relationship.""" - if src_grp: - src_meta = build_group_metadata(src_grp) - elif rel.type == GroupType.Identity: - # Fetch the supergroup (Version) of the Identity relations for metadata - src_meta = build_group_metadata(rel.source.supergroupsm2m[0].group) - else: - src_meta = build_group_metadata(rel.source) + if not src_grp: + if rel.type == GroupType.Identity: + # Fetch the supergroup (Version) of the Identity relations + src_grp = rel.source.supergroupsm2m[0].group + else: + src_grp = rel.source + src_meta = build_group_metadata(src_grp) + + if not trg_grp: + trg_grp = rel.target + trg_meta = build_group_metadata(trg_grp) + + if rel.type == GroupType.Identity: + rel_grp_int = rel.relation.value + # TODO: This this is the correct value, but there's a bug + # with superrelationshipsm2m being empty (shouldn't be the case) + # rel_grp_int = rel.superrelationshipsm2m[0].relationship_id.int - if trg_grp: - trg_meta = build_group_metadata(trg_grp) else: - trg_meta = build_group_metadata(rel.target) + rel_grp_int = rel.id.int + + # We deterministically recompute the document ID based on + # Source, Relationship and Target IDs + doc_uuid = uuid.UUID(int=(src_grp.id.int ^ rel_grp_int ^ trg_grp.id.int)) rel_meta = build_relationship_metadata(rel) grouping = grouping or \ ('identity' if rel.type == GroupType.Identity else 'version') return { - '_id': 'v:{}'.format(str(rel.id)), + '_id': str(doc_uuid), '_source': { "ID": str(rel.id), "Grouping": grouping, diff --git a/asclepias_broker/models.py b/asclepias_broker/models.py index 77428ec..9f0f6bb 100644 --- a/asclepias_broker/models.py +++ b/asclepias_broker/models.py @@ -433,9 +433,11 @@ class GroupRelationshipM2M(db.Model, Timestamp): nullable=False) relationship = orm_relationship(GroupRelationship, - foreign_keys=[relationship_id]) + foreign_keys=[relationship_id], + backref='subrelationshipsm2m') subrelationship = orm_relationship(GroupRelationship, - foreign_keys=[subrelationship_id]) + foreign_keys=[subrelationship_id], + backref='superrelationshipsm2m') def __repr__(self): """String representation of the model.""" From 79123f25cad9aea069a9e694deaf6bea297324fe Mon Sep 17 00:00:00 2001 From: Krzysztof Nowak Date: Fri, 4 May 2018 15:44:33 +0200 Subject: [PATCH 4/6] serializer: simplifies update type condition Signed-off-by: Krzysztof Nowak --- asclepias_broker/models.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/asclepias_broker/models.py b/asclepias_broker/models.py index 9f0f6bb..071a335 100644 --- a/asclepias_broker/models.py +++ b/asclepias_broker/models.py @@ -488,7 +488,7 @@ def update(self, payload, validate=True): new_json = deepcopy(self.json or {}) for k in OVERRIDABLE_KEYS: if payload.get(k): - if k == 'Type' and not _is_type_overridable(new_json, payload): + if k == 'Type' and not _is_type_overridable(payload): continue new_json[k] = payload[k] if validate: @@ -498,12 +498,8 @@ def update(self, payload, validate=True): return self -def _is_type_overridable(metadata, payload): - return not (metadata.get('Type') is not None and - metadata.get('Type') != {} and - metadata['Type'].get('Name') != 'unknown' and - (payload['Type'].get('Name') == 'unknown' or - payload['Type'].get('Name') == '')) +def _is_type_overridable(new_json): + return new_json.get('Type', {}).get('Name', 'unknown') != 'unknown' class GroupRelationshipMetadata(db.Model, Timestamp): From ef3ed98bf14502f23a827f394bbef94051d746ba Mon Sep 17 00:00:00 2001 From: Chiara Bigarella Date: Fri, 4 May 2018 15:48:58 +0200 Subject: [PATCH 5/6] views: skip force_https on /ping --- asclepias_broker/views.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/asclepias_broker/views.py b/asclepias_broker/views.py index 42ae64e..3b7dab1 100644 --- a/asclepias_broker/views.py +++ b/asclepias_broker/views.py @@ -20,6 +20,10 @@ from .errors import PayloadValidationRESTError from .models import Identifier +# +# UI Views +# + # TODO: we need this blueprint in order to run # asclepias_broker.tasks.process_event blueprint = Blueprint('asclepias_ui', __name__, template_folder='templates') @@ -31,6 +35,9 @@ def ping(): return 'OK' +ping.talisman_view_options = {'force_https': False} + + # # REST API Views # From c49a3424d5543e766035fb999e33f4668b9648c8 Mon Sep 17 00:00:00 2001 From: Krzysztof Nowak Date: Fri, 4 May 2018 16:13:03 +0200 Subject: [PATCH 6/6] tests: metadata test fix Signed-off-by: Krzysztof Nowak --- tests/model/test_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/model/test_metadata.py b/tests/model/test_metadata.py index d8a7d48..6bcebfe 100644 --- a/tests/model/test_metadata.py +++ b/tests/model/test_metadata.py @@ -85,13 +85,13 @@ def test_group_metadata_update_type(db): g = Group(type=GroupType.Identity) db.session.add(g) db.session.commit() - gm = GroupMetadata(group_id=g.id) + minimal_gm = {'Type': {'Name': 'unknown'}} + gm = GroupMetadata(group_id=g.id, json=minimal_gm) db.session.add(gm) db.session.commit() assert g.data == gm assert g.data.json == gm.json - update_and_compare(gm, {'Title': 'Some title'}) # Add Type update_and_compare(gm, {'Title': 'Some other title', 'Type': {'Name': 'unknown'}})