This repository has been archived by the owner on Jan 4, 2022. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2 from ogdch/develop
Release for fixes of latest support issues
- Loading branch information
Showing
2 changed files
with
196 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| from ckanext.harvest.harvesters import HarvesterBase | ||
|
|
||
| from sqlalchemy.sql import update,and_, bindparam | ||
| from sqlalchemy.exc import InvalidRequestError | ||
|
|
||
| from ckan import plugins as p | ||
| from ckan import model | ||
| from ckan.model import Session | ||
| from ckan.logic import ValidationError, NotFound, get_action | ||
| from ckan.logic.schema import default_create_package_schema | ||
| from ckan.lib.navl.validators import ignore_missing,ignore | ||
| from ckan.lib.munge import munge_tag | ||
|
|
||
| import logging | ||
| log = logging.getLogger(__name__) | ||
|
|
||
| class OGDCHHarvesterBase(HarvesterBase): | ||
| def _create_or_update_package(self, package_dict, harvest_object): | ||
| ''' | ||
| Creates a new package or updates an exisiting one according to the | ||
| package dictionary provided. The package dictionary should look like | ||
| the REST API response for a package: | ||
| http://ckan.net/api/rest/package/statistics-catalunya | ||
| Note that the package_dict must contain an id, which will be used to | ||
| check if the package needs to be created or updated (use the remote | ||
| dataset id). | ||
| If the remote server provides the modification date of the remote | ||
| package, add it to package_dict['metadata_modified']. | ||
| TODO: Not sure it is worth keeping this function. If useful it should | ||
| use the output of package_show logic function (maybe keeping support | ||
| for rest api based dicts | ||
| ''' | ||
| try: | ||
| # Change default schema | ||
| schema = default_create_package_schema() | ||
| schema['id'] = [ignore_missing, unicode] | ||
| schema['__junk'] = [ignore] | ||
|
|
||
| # Check API version | ||
| if self.config: | ||
| try: | ||
| api_version = int(self.config.get('api_version', 2)) | ||
| except ValueError: | ||
| raise ValueError('api_version must be an integer') | ||
|
|
||
| #TODO: use site user when available | ||
| user_name = self.config.get('user', u'harvest') | ||
| else: | ||
| api_version = 2 | ||
| user_name = u'harvest' | ||
|
|
||
| context = { | ||
| 'model': model, | ||
| 'session': Session, | ||
| 'user': user_name, | ||
| 'api_version': api_version, | ||
| 'schema': schema, | ||
| } | ||
|
|
||
| tags = package_dict.get('tags', []) | ||
| tags = [munge_tag(t) for t in tags] | ||
| tags = list(set(tags)) | ||
| package_dict['tags'] = tags | ||
|
|
||
| # Check if package exists | ||
| data_dict = {} | ||
| data_dict['id'] = package_dict['id'] | ||
| try: | ||
| existing_package_dict = get_action('package_show')(context, data_dict) | ||
| # Check modified date | ||
| if not 'metadata_modified' in package_dict or \ | ||
| package_dict['metadata_modified'] > existing_package_dict.get('metadata_modified'): | ||
| log.info('Package with GUID %s exists and needs to be updated' % harvest_object.guid) | ||
| # Update package | ||
| context.update({'id':package_dict['id']}) | ||
| new_package = get_action('package_update_rest')(context, package_dict) | ||
|
|
||
| else: | ||
| log.info('Package with GUID %s not updated, skipping...' % harvest_object.guid) | ||
| return | ||
|
|
||
| # Flag the other objects linking to this package as not current anymore | ||
| from ckanext.harvest.model import harvest_object_table | ||
| conn = Session.connection() | ||
| u = update(harvest_object_table) \ | ||
| .where(harvest_object_table.c.package_id==bindparam('b_package_id')) \ | ||
| .values(current=False) | ||
| conn.execute(u, b_package_id=new_package['id']) | ||
|
|
||
| # Flag this as the current harvest object | ||
|
|
||
| harvest_object.package_id = new_package['id'] | ||
| harvest_object.current = True | ||
| harvest_object.save() | ||
|
|
||
| except NotFound: | ||
| # Package needs to be created | ||
| log.info('Package with GUID %s does not exist, let\'s create it' % harvest_object.guid) | ||
| harvest_object.current = True | ||
| harvest_object.package_id = package_dict['id'] | ||
| # Defer constraints and flush so the dataset can be indexed with | ||
| # the harvest object id (on the after_show hook from the harvester | ||
| # plugin) | ||
| harvest_object.add() | ||
|
|
||
| model.Session.execute('SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED') | ||
| model.Session.flush() | ||
|
|
||
| new_package = get_action('package_create_rest')(context, package_dict) | ||
|
|
||
| Session.commit() | ||
|
|
||
| return True | ||
|
|
||
| except ValidationError,e: | ||
| log.exception(e) | ||
| self._save_object_error('Invalid package with GUID %s: %r'%(harvest_object.guid,e.error_dict),harvest_object,'Import') | ||
| except Exception, e: | ||
| log.exception(e) | ||
| self._save_object_error('%r'%e,harvest_object,'Import') | ||
|
|
||
| return None |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters