From da5c4362447d49da1f69b8b27625a5031201f536 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Sun, 28 Jul 2024 12:44:16 +0300 Subject: [PATCH] feat: remove missing packages from index during verification --- ckanext/federated_index/logic/action.py | 16 +++++++++++++++- ckanext/federated_index/logic/schema.py | 3 +++ ckanext/federated_index/tests/conftest.py | 5 ++++- .../federated_index/tests/logic/test_action.py | 17 +++++++++++++---- setup.cfg | 6 ++---- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/ckanext/federated_index/logic/action.py b/ckanext/federated_index/logic/action.py index cf952a6..56ca61c 100644 --- a/ckanext/federated_index/logic/action.py +++ b/ckanext/federated_index/logic/action.py @@ -10,6 +10,7 @@ import ckan.plugins.toolkit as tk from ckan import model from ckan.lib import search +from ckan.lib.search.query import solr_literal from ckan.logic import validate from ckanext.federated_index import config, interfaces, shared, storage @@ -56,6 +57,17 @@ def federated_index_profile_refresh( if data_dict["verify"]: for pkg_id in profile.check_ids(p["id"] for p in db.scan()): + mangled = db.get(pkg_id) + if not mangled: + continue + + for plugin in p.PluginImplementations(interfaces.IFederatedIndex): + mangled = plugin.federated_index_mangle_package(mangled, profile) # noqa: PLW2901 + tk.get_action("federated_index_profile_clear")( + tk.fresh_context(context), + {"profile": profile, "ids": [mangled["id"]]}, + ) + db.remove(pkg_id) for pkg in profile.fetch_packages(payload): @@ -119,7 +131,6 @@ def federated_index_profile_index( for pkg_dict in packages: for plugin in p.PluginImplementations(interfaces.IFederatedIndex): pkg_dict = plugin.federated_index_mangle_package(pkg_dict, profile) # noqa: PLW2901 - if model.Package.get(pkg_dict["name"]): log.warning("Package with name %s already exists", pkg_dict["name"]) continue @@ -184,6 +195,9 @@ def federated_index_profile_clear( conn = search.make_connection() query = f"+{config.profile_field()}:{profile.id}" + if ids := data_dict.get("ids"): + query += " id:({})".format(" OR ".join(map(solr_literal, ids))) + resp = conn.search(q=query, rows=0) conn.delete(q=query) diff --git a/ckanext/federated_index/logic/schema.py b/ckanext/federated_index/logic/schema.py index 1a955d3..71a171d 100644 --- a/ckanext/federated_index/logic/schema.py +++ b/ckanext/federated_index/logic/schema.py @@ -54,9 +54,12 @@ def profile_index( def profile_clear( not_empty: types.Validator, federated_index_profile: types.Validator, + json_list_or_string: types.Validator, + ignore_missing: types.Validator, ) -> types.Schema: return { "profile": [not_empty, federated_index_profile], + "ids": [ignore_missing, json_list_or_string], } diff --git a/ckanext/federated_index/tests/conftest.py b/ckanext/federated_index/tests/conftest.py index 7887aa8..a188bb9 100644 --- a/ckanext/federated_index/tests/conftest.py +++ b/ckanext/federated_index/tests/conftest.py @@ -26,7 +26,7 @@ def call_action( # noqa: PLR0913 return super().call_action(action, data_dict, context, apikey, files) -class TestFederatedIndexPlugin(p.Plugin): +class TestFederatedIndexPlugin(p.SingletonPlugin): p.implements(interfaces.IFederatedIndex, inherit=True) def federated_index_mangle_package( @@ -50,6 +50,9 @@ class TestProfile(shared.Profile): id: str = "test" url: str = "http://test.ckan.net" test_app: Any = None + extras: dict[str, Any] = dataclasses.field( + default_factory=lambda: {"search_payload": {"q": "-id:test-*"}} + ) def get_client(self): return TestAppCKAN(self.test_app) diff --git a/ckanext/federated_index/tests/logic/test_action.py b/ckanext/federated_index/tests/logic/test_action.py index c863762..2df8ba7 100644 --- a/ckanext/federated_index/tests/logic/test_action.py +++ b/ckanext/federated_index/tests/logic/test_action.py @@ -13,18 +13,27 @@ class TestRefresh: def test_verification(self, profile: shared.Profile, package_factory: Any): """Removed packages dropped from storage.""" - pkgs = package_factory.create_batch(3) - result = call_action("federated_index_profile_refresh", profile=profile) + result = call_action( + "federated_index_profile_refresh", profile=profile, index=True + ) assert result == {"count": 3, "profile": profile.id} + available = call_action("package_search", rows=0)["count"] + assert available == 6 call_action("package_delete", id=pkgs[-1]["id"]) result = call_action( - "federated_index_profile_refresh", profile=profile, verify=False + "federated_index_profile_refresh", profile=profile, verify=False, index=True ) assert result["count"] == 3 + available = call_action("package_search", rows=0)["count"] + assert available == 5 - result = call_action("federated_index_profile_refresh", profile=profile) + result = call_action( + "federated_index_profile_refresh", profile=profile, index=True + ) assert result["count"] == 2 + available = call_action("package_search", rows=0)["count"] + assert available == 4 diff --git a/setup.cfg b/setup.cfg index 08a4c7a..fe86661 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = ckanext-federated-index -version = 0.1.0 +version = 0.1.1 description = long_description = file: README.md long_description_content_type = text/markdown @@ -27,9 +27,7 @@ include_package_data = True [options.entry_points] ckan.plugins = federated_index = ckanext.federated_index.plugin:FederatedIndexPlugin - -ckan.test_plugins = - test_federated_index = ckanext.federated_index.tests.conftest:TestFederatedIndexPlugin + test_federated_index = ckanext.federated_index.tests.conftest:TestFederatedIndexPlugin babel.extractors = ckan = ckan.lib.extract:extract_ckan