From 9cd847f89e663fb6ae72d19c0ab257d2af8caabc Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:04:48 +0200 Subject: [PATCH 01/28] Clean commented lines --- ckanext/xloader/loader.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index 574f01d3..b4583472 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -50,13 +50,6 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): table_set = messytables.any_tableset(f, mimetype=mimetype, extension=extension) except messytables.ReadError as e: - # # try again with format - # f.seek(0) - # try: - # format = resource.get('format') - # table_set = messytables.any_tableset(f, mimetype=format, - # extension=format) - # except Exception: raise LoaderError('Messytables error: {}'.format(e)) except Exception as e: raise FileCouldNotBeLoadedError(e) From 742ccb2ab76b2657cba18162ec7925df4da0d751 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:15:40 +0200 Subject: [PATCH 02/28] Clean old commented lines --- ckanext/xloader/loader.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ckanext/xloader/loader.py b/ckanext/xloader/loader.py index b4583472..01a10224 100644 --- a/ckanext/xloader/loader.py +++ b/ckanext/xloader/loader.py @@ -82,8 +82,6 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # types = messytables.type_guess(row_set.sample, types=TYPES, strict=True) headers = [header.strip()[:MAX_COLUMN_LENGTH] for header in headers if header.strip()] - # headers_dicts = [dict(id=field[0], type=TYPE_MAPPING[str(field[1])]) - # for field in zip(headers, types)] # TODO worry about csv header name problems # e.g. duplicate names @@ -101,8 +99,6 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): f_write.close() # ensures the last line is written csv_filepath = f_write.name - # check tables exists - # datastore db connection engine = get_write_engine() @@ -172,10 +168,7 @@ def load_csv(csv_filepath, resource_id, mimetype='text/csv', logger=None): # datstore_active is switched on by datastore_create - TODO temporarily # disable it until the load is complete - - # logger.info('Disabling row index trigger') _disable_fulltext_trigger(connection, resource_id) - # logger.info('Dropping indexes') _drop_indexes(context, data_dict, False) logger.info('Copying to database...') @@ -356,10 +349,6 @@ def row_iterator(): logger.info('Determined headers and types: {headers}'.format( headers=headers_dicts)) - # Commented - this is only for tests - # if dry_run: - # return headers_dicts, result - logger.info('Copying to database...') count = 0 for i, records in enumerate(chunky(result, 250)): @@ -375,10 +364,6 @@ def row_iterator(): # no datastore table is created raise LoaderError('No entries found - nothing to load') - # Commented - this is done by the caller in jobs.py - # if data.get('set_url_type', False): - # update_resource(resource, api_key, ckan_url) - _TYPE_MAPPING = { 'String': 'text', @@ -393,8 +378,6 @@ def row_iterator(): def get_types(): _TYPES = [messytables.StringType, messytables.DecimalType, messytables.IntegerType, messytables.DateUtilType] - # TODO make this configurable - # TYPES = web.app.config.get('TYPES', _TYPES) TYPE_MAPPING = config.get('TYPE_MAPPING', _TYPE_MAPPING) return _TYPES, TYPE_MAPPING From e113bedbae91153f8e56393eb128fed59e735bc6 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:23:24 +0200 Subject: [PATCH 03/28] We are no longer using the task so no use on calling it --- ckanext/xloader/plugin.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index aa42f5e8..38de3b67 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -150,23 +150,6 @@ def _submit_to_xloader(self, resource_dict): 'would be circular.'.format(**resource_dict)) return - try: - task = p.toolkit.get_action('task_status_show')( - context, { - 'entity_id': resource_dict["id"], - 'task_type': 'xloader', - 'key': 'xloader'} - ) - # if task.get('state') == 'pending': - # # There already is a pending DataPusher submission, - # # skip this one ... - # log.debug( - # 'Skipping DataPusher submission for ' - # 'resource {0}'.format(entity.id)) - # return - except p.toolkit.ObjectNotFound: - pass - try: log.debug('Submitting resource {0} to be xloadered' .format(resource_dict["id"])) From 572a5578331d977bd544531e1f8e758488b3e752 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:27:43 +0200 Subject: [PATCH 04/28] Organize imports --- ckanext/xloader/plugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index 38de3b67..e3afa496 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -1,13 +1,15 @@ +import logging + from ckan import model +from ckan.common import config import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit -from ckan.common import config from ckanext.xloader import action, auth import ckanext.xloader.helpers as xloader_helpers from ckanext.xloader.loader import fulltext_function_exists, get_write_engine -log = __import__('logging').getLogger(__name__) +log = logging.getLogger(__name__) p = plugins From 3cb139d14a47678e6de5871583eeee823fcbdbf4 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:31:14 +0200 Subject: [PATCH 05/28] Clean unnecessary re-assignment --- ckanext/xloader/plugin.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index e3afa496..eb93afe4 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -10,7 +10,6 @@ from ckanext.xloader.loader import fulltext_function_exists, get_write_engine log = logging.getLogger(__name__) -p = plugins # resource.formats accepted by ckanext-xloader. Must be lowercase here. @@ -76,7 +75,7 @@ def before_map(self, m): def update_config(self, config): templates_base = config.get('ckan.base_templates_folder', 'templates-bs2') # for ckan < 2.8 - p.toolkit.add_template_directory(config, templates_base) + toolkit.add_template_directory(config, templates_base) # IConfigurable @@ -92,7 +91,7 @@ def configure(self, config_): 'Config option `{0}` must be set to use ckanext-xloader.' .format(config_option)) - if p.toolkit.check_ckan_version(max_version='2.7.99'): + if toolkit.check_ckan_version(max_version='2.7.99'): # populate_full_text_trigger() needs to be defined, and this was # introduced in CKAN 2.8 when you installed datastore e.g.: # paster datastore set-permissions @@ -155,11 +154,11 @@ def _submit_to_xloader(self, resource_dict): try: log.debug('Submitting resource {0} to be xloadered' .format(resource_dict["id"])) - p.toolkit.get_action('xloader_submit')(context, { + toolkit.get_action('xloader_submit')(context, { 'resource_id': resource_dict["id"], 'ignore_hash': self.ignore_hash, }) - except p.toolkit.ValidationError as e: + except toolkit.ValidationError as e: # If xloader is offline, we want to catch error instead # of raising otherwise resource save will fail with 500 log.critical(e) From 19a4cf254cd12daa87cde4413febd4b4f1a403f9 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:33:34 +0200 Subject: [PATCH 06/28] Formating of plugin.py for better readability --- ckanext/xloader/plugin.py | 133 +++++++++++++++++++++++--------------- 1 file changed, 81 insertions(+), 52 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index eb93afe4..b51d99e2 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -14,11 +14,15 @@ # resource.formats accepted by ckanext-xloader. Must be lowercase here. DEFAULT_FORMATS = [ - 'csv', 'application/csv', - 'xls', 'xlsx', 'tsv', - 'application/vnd.ms-excel', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', - 'ods', 'application/vnd.oasis.opendocument.spreadsheet', + "csv", + "application/csv", + "xls", + "xlsx", + "tsv", + "application/vnd.ms-excel", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "ods", + "application/vnd.oasis.opendocument.spreadsheet", ] @@ -28,7 +32,7 @@ class XLoaderFormats(object): @classmethod def is_it_an_xloader_format(cls, format_): if cls.formats is None: - cls._formats = config.get('ckanext.xloader.formats') + cls._formats = config.get("ckanext.xloader.formats") if cls._formats is not None: cls._formats = cls._formats.lower().split() else: @@ -47,51 +51,61 @@ class xloaderPlugin(plugins.SingletonPlugin): plugins.implements(plugins.ITemplateHelpers) plugins.implements(plugins.IResourceController, inherit=True) - if toolkit.check_ckan_version('2.9'): + if toolkit.check_ckan_version("2.9"): plugins.implements(plugins.IClick) plugins.implements(plugins.IBlueprint) # IClick def get_commands(self): from ckanext.xloader.cli import get_commands + return get_commands() + # IBlueprint def get_blueprint(self): from ckanext.xloader.views import get_blueprints + return get_blueprints() + else: plugins.implements(plugins.IRoutes, inherit=True) # IRoutes def before_map(self, m): m.connect( - 'xloader.resource_data', '/dataset/{id}/resource_data/{resource_id}', - controller='ckanext.xloader.controllers:ResourceDataController', - action='resource_data', ckan_icon='cloud-upload') + "xloader.resource_data", + "/dataset/{id}/resource_data/{resource_id}", + controller="ckanext.xloader.controllers:ResourceDataController", + action="resource_data", + ckan_icon="cloud-upload", + ) return m # IConfigurer def update_config(self, config): - templates_base = config.get('ckan.base_templates_folder', - 'templates-bs2') # for ckan < 2.8 + templates_base = config.get( + "ckan.base_templates_folder", "templates-bs2" + ) # for ckan < 2.8 toolkit.add_template_directory(config, templates_base) # IConfigurable def configure(self, config_): - if config_.get('ckanext.xloader.ignore_hash') in ['True', 'TRUE', '1', True, 1]: + if config_.get("ckanext.xloader.ignore_hash") in ["True", "TRUE", "1", True, 1]: self.ignore_hash = True else: self.ignore_hash = False - for config_option in ('ckan.site_url',): + for config_option in ("ckan.site_url",): if not config_.get(config_option): raise Exception( - 'Config option `{0}` must be set to use ckanext-xloader.' - .format(config_option)) + "Config option `{0}` must be set to use ckanext-xloader.".format( + config_option + ) + ) - if toolkit.check_ckan_version(max_version='2.7.99'): + if toolkit.check_ckan_version(max_version="2.7.99"): # populate_full_text_trigger() needs to be defined, and this was # introduced in CKAN 2.8 when you installed datastore e.g.: # paster datastore set-permissions @@ -99,9 +113,11 @@ def configure(self, config_): # this function manually. connection = get_write_engine().connect() if not fulltext_function_exists(connection): - raise Exception('populate_full_text_trigger is not defined. ' - 'See ckanext-xloader\'s README.rst for more ' - 'details.') + raise Exception( + "populate_full_text_trigger is not defined. " + "See ckanext-xloader's README.rst for more " + "details." + ) # IResourceUrlChange @@ -111,53 +127,67 @@ def notify(self, resource): "ignore_auth": True, } resource_dict = toolkit.get_action("resource_show")( - context, { + context, + { "id": resource.id, - } + }, ) self._submit_to_xloader(resource_dict) # IResourceController - if toolkit.check_ckan_version('2.10'): + if toolkit.check_ckan_version("2.10"): + def after_resource_create(self, context, resource_dict): self._submit_to_xloader(resource_dict) def before_resource_show(self, resource_dict): resource_dict[ - 'datastore_contains_all_records_of_source_file'] = toolkit.asbool( - resource_dict.get('datastore_contains_all_records_of_source_file')) + "datastore_contains_all_records_of_source_file" + ] = toolkit.asbool( + resource_dict.get("datastore_contains_all_records_of_source_file") + ) + else: + def after_create(self, context, resource_dict): self._submit_to_xloader(resource_dict) def before_show(self, resource_dict): resource_dict[ - 'datastore_contains_all_records_of_source_file'] = toolkit.asbool( - resource_dict.get('datastore_contains_all_records_of_source_file')) + "datastore_contains_all_records_of_source_file" + ] = toolkit.asbool( + resource_dict.get("datastore_contains_all_records_of_source_file") + ) def _submit_to_xloader(self, resource_dict): - context = {'model': model, 'ignore_auth': True, - 'defer_commit': True} + context = {"model": model, "ignore_auth": True, "defer_commit": True} if not XLoaderFormats.is_it_an_xloader_format(resource_dict["format"]): - log.debug('Skipping xloading resource {id} because ' - 'format "{format}" is not configured to be ' - 'xloadered' - .format(**resource_dict)) + log.debug( + "Skipping xloading resource {id} because " + 'format "{format}" is not configured to be ' + "xloadered".format(**resource_dict) + ) return - if resource_dict["url_type"] in ('datapusher', 'xloader'): - log.debug('Skipping xloading resource {id} because ' - 'url_type "{url_type}" means resource.url ' - 'points to the datastore already, so loading ' - 'would be circular.'.format(**resource_dict)) + if resource_dict["url_type"] in ("datapusher", "xloader"): + log.debug( + "Skipping xloading resource {id} because " + 'url_type "{url_type}" means resource.url ' + "points to the datastore already, so loading " + "would be circular.".format(**resource_dict) + ) return try: - log.debug('Submitting resource {0} to be xloadered' - .format(resource_dict["id"])) - toolkit.get_action('xloader_submit')(context, { - 'resource_id': resource_dict["id"], - 'ignore_hash': self.ignore_hash, - }) + log.debug( + "Submitting resource {0} to be xloadered".format(resource_dict["id"]) + ) + toolkit.get_action("xloader_submit")( + context, + { + "resource_id": resource_dict["id"], + "ignore_hash": self.ignore_hash, + }, + ) except toolkit.ValidationError as e: # If xloader is offline, we want to catch error instead # of raising otherwise resource save will fail with 500 @@ -168,24 +198,23 @@ def _submit_to_xloader(self, resource_dict): def get_actions(self): return { - 'xloader_submit': action.xloader_submit, - 'xloader_hook': action.xloader_hook, - 'xloader_status': action.xloader_status, + "xloader_submit": action.xloader_submit, + "xloader_hook": action.xloader_hook, + "xloader_status": action.xloader_status, } # IAuthFunctions def get_auth_functions(self): return { - 'xloader_submit': auth.xloader_submit, - 'xloader_status': auth.xloader_status, + "xloader_submit": auth.xloader_submit, + "xloader_status": auth.xloader_status, } # ITemplateHelpers def get_helpers(self): return { - 'xloader_status': xloader_helpers.xloader_status, - 'xloader_status_description': - xloader_helpers.xloader_status_description, + "xloader_status": xloader_helpers.xloader_status, + "xloader_status_description": xloader_helpers.xloader_status_description, } From ff1f394b39c9f77b7c701d87e1bec828f8d011ce Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:34:54 +0200 Subject: [PATCH 07/28] model object is prepopulated by ckan --- ckanext/xloader/plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index b51d99e2..b15e3d9b 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -1,6 +1,5 @@ import logging -from ckan import model from ckan.common import config import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit @@ -123,7 +122,6 @@ def configure(self, config_): def notify(self, resource): context = { - "model": model, "ignore_auth": True, } resource_dict = toolkit.get_action("resource_show")( @@ -160,7 +158,7 @@ def before_show(self, resource_dict): ) def _submit_to_xloader(self, resource_dict): - context = {"model": model, "ignore_auth": True, "defer_commit": True} + context = {"ignore_auth": True, "defer_commit": True} if not XLoaderFormats.is_it_an_xloader_format(resource_dict["format"]): log.debug( "Skipping xloading resource {id} because " From 12616ac1e5e9651380672003738994b4f817294a Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:47:29 +0200 Subject: [PATCH 08/28] No usage for variables in c object --- ckanext/xloader/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index f2800d54..58d56bac 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -5,7 +5,7 @@ def resource_data(id, resource_id): if p.toolkit.request.method == "POST": try: - p.toolkit.c.pkg_dict = p.toolkit.get_action("xloader_submit")( + p.toolkit.get_action("xloader_submit")( None, { "resource_id": resource_id, "ignore_hash": True, # user clicked the reload button @@ -19,8 +19,8 @@ def resource_data(id, resource_id): ) try: - p.toolkit.c.pkg_dict = p.toolkit.get_action("package_show")(None, {"id": id}) - p.toolkit.c.resource = p.toolkit.get_action("resource_show")( + pkg_dict = p.toolkit.get_action("package_show")(None, {"id": id}) + resource = p.toolkit.get_action("resource_show")( None, {"id": resource_id} ) except (p.toolkit.ObjectNotFound, p.toolkit.NotAuthorized): @@ -39,7 +39,7 @@ def resource_data(id, resource_id): "xloader/resource_data.html", extra_vars={ "status": xloader_status, - "resource": p.toolkit.c.resource, - "pkg_dict": p.toolkit.c.pkg_dict, + "resource": resource, + "pkg_dict": pkg_dict, }, ) From 696c761030bd1cf860f9ffbc6045594a6a68bd25 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:52:30 +0200 Subject: [PATCH 09/28] Small formatting --- ckanext/xloader/utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index 58d56bac..a8f9a7f3 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -6,10 +6,11 @@ def resource_data(id, resource_id): if p.toolkit.request.method == "POST": try: p.toolkit.get_action("xloader_submit")( - None, { + None, + { "resource_id": resource_id, "ignore_hash": True, # user clicked the reload button - } + }, ) except p.toolkit.ValidationError: pass @@ -20,9 +21,7 @@ def resource_data(id, resource_id): try: pkg_dict = p.toolkit.get_action("package_show")(None, {"id": id}) - resource = p.toolkit.get_action("resource_show")( - None, {"id": resource_id} - ) + resource = p.toolkit.get_action("resource_show")(None, {"id": resource_id}) except (p.toolkit.ObjectNotFound, p.toolkit.NotAuthorized): return p.toolkit.abort(404, p.toolkit._("Resource not found")) From 49f35f8134b83fbe0705cb5d5e236804f742d8f8 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 10:57:22 +0200 Subject: [PATCH 10/28] Organize imports --- ckanext/xloader/action.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index b3e96d7b..71867a24 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -1,20 +1,20 @@ # encoding: utf-8 from __future__ import absolute_import -from six import text_type as str -import logging -import json import datetime +import json +import logging -from dateutil.parser import parse as parse_date - -import ckan.lib.navl.dictization_functions from ckan import logic -import ckan.plugins as p -from ckan.logic import side_effect_free import ckan.lib.jobs as rq_jobs +import ckan.lib.navl.dictization_functions +from ckan.logic import side_effect_free +import ckan.plugins as p +from dateutil.parser import parse as parse_date +from six import text_type as str import ckanext.xloader.schema + from . import interfaces as xloader_interfaces from . import jobs from . import db From 0c56422b7d50f92e31bec98715cb790a71d36425 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:02:02 +0200 Subject: [PATCH 11/28] Do not import ckan logic directly --- ckanext/xloader/action.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 71867a24..120127d8 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -5,7 +5,6 @@ import json import logging -from ckan import logic import ckan.lib.jobs as rq_jobs import ckan.lib.navl.dictization_functions from ckan.logic import side_effect_free @@ -25,7 +24,7 @@ log = logging.getLogger(__name__) config = p.toolkit.config -_get_or_bust = logic.get_or_bust +_get_or_bust = p.toolkit.get_or_bust _validate = ckan.lib.navl.dictization_functions.validate @@ -62,7 +61,7 @@ def xloader_submit(context, data_dict): resource_dict = p.toolkit.get_action('resource_show')(context, { 'id': res_id, }) - except logic.NotFound: + except p.toolkit.NotFound: return False site_url = config['ckan.site_url'] @@ -138,7 +137,7 @@ def xloader_submit(context, data_dict): return False task['id'] = existing_task['id'] - except logic.NotFound: + except p.toolkit.NotFound: pass model = context['model'] @@ -275,7 +274,7 @@ def xloader_hook(context, data_dict): for plugin in p.PluginImplementations(xloader_interfaces.IXloader): plugin.after_upload(context, resource_dict, dataset_dict) - logic.get_action('resource_create_default_resource_views')( + p.toolkit.get_action('resource_create_default_resource_views')( context, { 'resource': resource_dict, From 281c8c443bb8a21258d17d7ec58932c4157c5304 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:10:55 +0200 Subject: [PATCH 12/28] Move assignment close to use of variable --- ckanext/xloader/action.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 120127d8..2c0cde68 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -53,10 +53,9 @@ def xloader_submit(context, data_dict): if errors: raise p.toolkit.ValidationError(errors) - res_id = data_dict['resource_id'] - p.toolkit.check_access('xloader_submit', context, data_dict) + res_id = data_dict['resource_id'] try: resource_dict = p.toolkit.get_action('resource_show')(context, { 'id': res_id, @@ -64,16 +63,6 @@ def xloader_submit(context, data_dict): except p.toolkit.NotFound: return False - site_url = config['ckan.site_url'] - callback_url = p.toolkit.url_for( - "api.action", - ver=3, - logic_function="xloader_hook", - qualified=True - ) - - site_user = p.toolkit.get_action('get_site_user')({'ignore_auth': True}, {}) - for plugin in p.PluginImplementations(xloader_interfaces.IXloader): upload = plugin.can_upload(res_id) if not upload: @@ -149,13 +138,20 @@ def xloader_submit(context, data_dict): task ) + site_user = p.toolkit.get_action('get_site_user')({'ignore_auth': True}, {}) + callback_url = p.toolkit.url_for( + "api.action", + ver=3, + logic_function="xloader_hook", + qualified=True + ) data = { 'api_key': site_user['apikey'], 'job_type': 'xloader_to_datastore', 'result_url': callback_url, 'metadata': { 'ignore_hash': data_dict.get('ignore_hash', False), - 'ckan_url': site_url, + 'ckan_url': config['ckan.site_url'], 'resource_id': res_id, 'set_url_type': data_dict.get('set_url_type', False), 'task_created': task['last_updated'], From 2b7ef80c140d88a9f0a898d2fd3261f014c96db4 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:11:48 +0200 Subject: [PATCH 13/28] Fix toolkit Exception --- ckanext/xloader/action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 2c0cde68..4b34d84a 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -60,7 +60,7 @@ def xloader_submit(context, data_dict): resource_dict = p.toolkit.get_action('resource_show')(context, { 'id': res_id, }) - except p.toolkit.NotFound: + except p.toolkit.ObjectNotFound: return False for plugin in p.PluginImplementations(xloader_interfaces.IXloader): @@ -126,7 +126,7 @@ def xloader_submit(context, data_dict): return False task['id'] = existing_task['id'] - except p.toolkit.NotFound: + except p.toolkit.ObjectNotFound: pass model = context['model'] From 6356d6e9f6ab21e14b3707b22b665292cb3c7f1d Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:35:35 +0200 Subject: [PATCH 14/28] Drop timeout support for older CKANs --- ckanext/xloader/action.py | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 4b34d84a..a98f72de 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -160,12 +160,9 @@ def xloader_submit(context, data_dict): } timeout = config.get('ckanext.xloader.job_timeout', '3600') try: - try: - job = enqueue_job(jobs.xloader_data_into_datastore, [data], - timeout=timeout) - except TypeError: - # older ckans didn't allow the timeout keyword - job = _enqueue(jobs.xloader_data_into_datastore, [data], timeout=timeout) + job = enqueue_job( + jobs.xloader_data_into_datastore, [data], timeout=timeout + ) except Exception: log.exception('Unable to enqueued xloader res_id=%s', res_id) return False @@ -187,26 +184,6 @@ def xloader_submit(context, data_dict): return True -def _enqueue(fn, args=None, kwargs=None, title=None, queue='default', - timeout=180): - '''Same as latest ckan.lib.jobs.enqueue - earlier CKAN versions dont have - the timeout param''' - if args is None: - args = [] - if kwargs is None: - kwargs = {} - job = get_queue(queue).enqueue_call(func=fn, args=args, kwargs=kwargs, - timeout=timeout) - job.meta[u'title'] = title - job.save() - msg = u'Added background job {}'.format(job.id) - if title: - msg = u'{} ("{}")'.format(msg, title) - msg = u'{} to queue "{}"'.format(msg, queue) - log.info(msg) - return job - - def xloader_hook(context, data_dict): ''' Update xloader task. This action is typically called by ckanext-xloader whenever the status of a job changes. From 53b93d18c0d612206371d4edd69e2a59deac71a9 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:37:38 +0200 Subject: [PATCH 15/28] Small formatting --- ckanext/xloader/action.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index a98f72de..6c987b9a 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -131,12 +131,10 @@ def xloader_submit(context, data_dict): model = context['model'] - p.toolkit.get_action('task_status_update')({ - 'session': model.meta.create_local_session(), - 'ignore_auth': True - }, + p.toolkit.get_action('task_status_update')( + {'session': model.meta.create_local_session(), 'ignore_auth': True}, task - ) + ) site_user = p.toolkit.get_action('get_site_user')({'ignore_auth': True}, {}) callback_url = p.toolkit.url_for( @@ -174,12 +172,10 @@ def xloader_submit(context, data_dict): task['state'] = 'pending' task['last_updated'] = str(datetime.datetime.utcnow()) - p.toolkit.get_action('task_status_update')({ - 'session': model.meta.create_local_session(), - 'ignore_auth': True - }, + p.toolkit.get_action('task_status_update')( + {'session': model.meta.create_local_session(), 'ignore_auth': True}, task - ) + ) return True From d369e0eb057cc68f434c4c93a352c2930341a263 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 11:47:34 +0200 Subject: [PATCH 16/28] Drop old support for id variable --- ckanext/xloader/action.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 6c987b9a..a4e77013 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -293,8 +293,6 @@ def xloader_status(context, data_dict): p.toolkit.check_access('xloader_status', context, data_dict) - if 'id' in data_dict: - data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') task = p.toolkit.get_action('task_status_show')(context, { From 539a6c22f01147d81c64955b68ac9c3d985c9fe3 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Tue, 28 Jun 2022 15:26:29 +0200 Subject: [PATCH 17/28] Fix enqueue_job call --- ckanext/xloader/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index a4e77013..84855bf4 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -159,7 +159,7 @@ def xloader_submit(context, data_dict): timeout = config.get('ckanext.xloader.job_timeout', '3600') try: job = enqueue_job( - jobs.xloader_data_into_datastore, [data], timeout=timeout + jobs.xloader_data_into_datastore, [data], rq_kwargs=dict(timeout=timeout) ) except Exception: log.exception('Unable to enqueued xloader res_id=%s', res_id) From fd2899ea628c51125eb0fd2db2fb4b83183015a6 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 29 Jun 2022 13:43:48 +0200 Subject: [PATCH 18/28] Add support for API Token in xloader_submit --- ckanext/xloader/action.py | 4 ++-- ckanext/xloader/tests/test_action.py | 15 ++++++++++++++- ckanext/xloader/utils.py | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 84855bf4..3268660a 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -17,6 +17,7 @@ from . import interfaces as xloader_interfaces from . import jobs from . import db +from . import utils enqueue_job = p.toolkit.enqueue_job get_queue = rq_jobs.get_queue @@ -136,7 +137,6 @@ def xloader_submit(context, data_dict): task ) - site_user = p.toolkit.get_action('get_site_user')({'ignore_auth': True}, {}) callback_url = p.toolkit.url_for( "api.action", ver=3, @@ -144,7 +144,7 @@ def xloader_submit(context, data_dict): qualified=True ) data = { - 'api_key': site_user['apikey'], + 'api_key': utils.get_xloader_user_apitoken(), 'job_type': 'xloader_to_datastore', 'result_url': callback_url, 'metadata': { diff --git a/ckanext/xloader/tests/test_action.py b/ckanext/xloader/tests/test_action.py index d6e7074e..3e551276 100644 --- a/ckanext/xloader/tests/test_action.py +++ b/ckanext/xloader/tests/test_action.py @@ -3,6 +3,8 @@ from ckan.tests import helpers, factories +from ckanext.xloader.utils import get_xloader_user_apitoken + @pytest.mark.usefixtures("clean_db", "with_plugins") @pytest.mark.ckan_config("ckan.plugins", "datastore xloader") @@ -90,4 +92,15 @@ def test_status(self): resource_id=res["id"], ) - assert status['status'] == 'pending' + assert status["status"] == "pending" + + def test_xloader_user_api_token_defaults_to_site_user_apikey(self): + api_token = get_xloader_user_apitoken() + site_user = helpers.call_action("get_site_user") + assert api_token == site_user["apikey"] + + @pytest.mark.ckan_config("ckanext.xloader.api_token", "random-api-token") + def test_xloader_user_api_token(self): + api_token = get_xloader_user_apitoken() + + assert api_token == "random-api-token" diff --git a/ckanext/xloader/utils.py b/ckanext/xloader/utils.py index a8f9a7f3..43fbd4e2 100644 --- a/ckanext/xloader/utils.py +++ b/ckanext/xloader/utils.py @@ -42,3 +42,18 @@ def resource_data(id, resource_id): "pkg_dict": pkg_dict, }, ) + + +def get_xloader_user_apitoken(): + """ Returns the API Token for authentication. + + xloader actions require an authenticated user to perform the actions. This + method returns the api_token set in the config file and defaults to the + site_user. + """ + api_token = p.toolkit.config.get('ckanext.xloader.api_token', None) + if api_token: + return api_token + + site_user = p.toolkit.get_action('get_site_user')({'ignore_auth': True}, {}) + return site_user["apikey"] From 7581feee2d0856f721c345a182e38b6d3d7f9786 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 29 Jun 2022 13:45:41 +0200 Subject: [PATCH 19/28] Add new config to README.md --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 2212a64c..0cb136e6 100644 --- a/README.rst +++ b/README.rst @@ -265,6 +265,11 @@ Configuration: # to True. ckanext.xloader.ssl_verify = True + # Uses a specific API token for the xloader_submit action instead of the + # apikey of the site_user + ckanext.xloader.api_token = ckan-provided-api-token + + ------------------------ Developer installation ------------------------ From e9cda7803cec1249c7033421d95515479264e643 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Wed, 29 Jun 2022 15:09:00 +0200 Subject: [PATCH 20/28] Model is prepopulated by toolkit --- ckanext/xloader/command.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/ckanext/xloader/command.py b/ckanext/xloader/command.py index db6f8117..c5ca5212 100644 --- a/ckanext/xloader/command.py +++ b/ckanext/xloader/command.py @@ -23,17 +23,16 @@ def _setup_xloader_logger(self): logger.propagate = False # in case the config def _submit_all_existing(self): - import ckan.model as model from ckanext.datastore.backend \ import get_all_resources_ids_in_datastore resource_ids = get_all_resources_ids_in_datastore() print('Processing %d resources' % len(resource_ids)) user = tk.get_action('get_site_user')( - {'model': model, 'ignore_auth': True}, {}) + {'ignore_auth': True}, {}) for resource_id in resource_ids: try: resource_dict = tk.get_action('resource_show')( - {'model': model, 'ignore_auth': True}, {'id': resource_id}) + {'ignore_auth': True}, {'id': resource_id}) except tk.ObjectNotFound: print(' Skipping resource {} found in datastore but not in ' 'metadata'.format(resource_id)) @@ -44,25 +43,23 @@ def _submit_all(self): # submit every package # for each package in the package list, # submit each resource w/ _submit_package - import ckan.model as model package_list = tk.get_action('package_search')( - {'model': model, 'ignore_auth': True}, {'include_private': True, 'rows': 1000}) + {'ignore_auth': True}, {'include_private': True, 'rows': 1000}) package_list = [pkg['id'] for pkg in package_list['results']] print('Processing %d datasets' % len(package_list)) user = tk.get_action('get_site_user')( - {'model': model, 'ignore_auth': True}, {}) + {'ignore_auth': True}, {}) for p_id in package_list: self._submit_package(p_id, user, indent=2) def _submit_package(self, pkg_id, user=None, indent=0): - import ckan.model as model if not user: user = tk.get_action('get_site_user')( - {'model': model, 'ignore_auth': True}, {}) + {'ignore_auth': True}, {}) try: pkg = tk.get_action('package_show')( - {'model': model, 'ignore_auth': True}, + {'ignore_auth': True}, {'id': pkg_id.strip()}) except Exception as e: print(e) From e454e791bd7cc650e199c9306e1f8fd2f956e64e Mon Sep 17 00:00:00 2001 From: Patricio Del Boca Date: Thu, 30 Jun 2022 09:14:10 +0200 Subject: [PATCH 21/28] Update ckanext/xloader/plugin.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrià Mercader --- ckanext/xloader/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/xloader/plugin.py b/ckanext/xloader/plugin.py index b15e3d9b..6b0c8e0e 100644 --- a/ckanext/xloader/plugin.py +++ b/ckanext/xloader/plugin.py @@ -1,6 +1,6 @@ import logging -from ckan.common import config +from ckan.plugins.toolkit import config import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit From 68bc38f43530bf5d1679cdc4aa98448a112a3db2 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 30 Jun 2022 09:22:09 +0200 Subject: [PATCH 22/28] Restore support for 2.7 --- ckanext/xloader/action.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 3268660a..9258b39f 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -161,6 +161,11 @@ def xloader_submit(context, data_dict): job = enqueue_job( jobs.xloader_data_into_datastore, [data], rq_kwargs=dict(timeout=timeout) ) + except TypeError: + # This except provides support for 2.7. + job = enqueue_job( + jobs.xloader_data_into_datastore, [data], timeout=timeout + ) except Exception: log.exception('Unable to enqueued xloader res_id=%s', res_id) return False From 56402f1de8b0e3b641bb6bcdca9a30f7425785e3 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 30 Jun 2022 09:29:33 +0200 Subject: [PATCH 23/28] Restore _enqueue method --- ckanext/xloader/action.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ckanext/xloader/action.py b/ckanext/xloader/action.py index 9258b39f..ff235614 100644 --- a/ckanext/xloader/action.py +++ b/ckanext/xloader/action.py @@ -163,7 +163,7 @@ def xloader_submit(context, data_dict): ) except TypeError: # This except provides support for 2.7. - job = enqueue_job( + job = _enqueue( jobs.xloader_data_into_datastore, [data], timeout=timeout ) except Exception: @@ -185,6 +185,29 @@ def xloader_submit(context, data_dict): return True +def _enqueue(fn, args=None, kwargs=None, title=None, queue='default', + timeout=180): + '''Same as latest ckan.lib.jobs.enqueue - earlier CKAN versions dont have + the timeout param + + This function can be removed when dropping support for 2.7 + ''' + if args is None: + args = [] + if kwargs is None: + kwargs = {} + job = get_queue(queue).enqueue_call(func=fn, args=args, kwargs=kwargs, + timeout=timeout) + job.meta[u'title'] = title + job.save() + msg = u'Added background job {}'.format(job.id) + if title: + msg = u'{} ("{}")'.format(msg, title) + msg = u'{} to queue "{}"'.format(msg, queue) + log.info(msg) + return job + + def xloader_hook(context, data_dict): ''' Update xloader task. This action is typically called by ckanext-xloader whenever the status of a job changes. From ed2df70dabc17530f8e04203a9e2f3c5c34f83d7 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 30 Jun 2022 15:17:49 +0200 Subject: [PATCH 24/28] Support API Token to get resource and dataset --- ckanext/xloader/jobs.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 90411119..154f0f08 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -17,8 +17,9 @@ import sqlalchemy as sa import ckan.model as model -from ckan.plugins.toolkit import get_action, asbool, ObjectNotFound, config +from ckan.plugins.toolkit import get_action, asbool, ObjectNotFound, config, check_ckan_version import ckan.lib.search as search +from ckan.lib.api_token import get_user_from_token from . import loader from . import db @@ -480,16 +481,45 @@ def update_resource(resource, patch_only=False): get_action(action)(context, resource) +def _get_user_from_key(api_key_or_token): + """ Gets the user using the API Token or API Key. + + This method provides backwards compatibility for CKAN 2.9 that + supported both methods and previous CKAN versions supporting + only API Keys. + """ + user = None + + if check_ckan_version(min_version="2.10"): + user = get_user_from_token(api_key_or_token) + return user + + if check_ckan_version(min_version="2.9"): + user = get_user_from_token(api_key_or_token) + if not user: + user = model.Session.query(model.User).filter_by( + apikey=api_key_or_token + ).first() + return user + + if check_ckan_version(min_version="2.7"): + user = model.Session.query(model.User).filter_by( + apikey=api_key_or_token + ).first() + return user + + return user + + def get_resource_and_dataset(resource_id, api_key): """ Gets available information about the resource and its dataset from CKAN """ - user = model.Session.query(model.User).filter_by( - apikey=api_key).first() + context = None + user = _get_user_from_key(api_key) if user is not None: context = {'user': user.name} - else: - context = None + res_dict = get_action('resource_show')(context, {'id': resource_id}) pkg_dict = get_action('package_show')(context, {'id': res_dict['package_id']}) return res_dict, pkg_dict From 296b9816494ec9a335844feb575686b651c08a20 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 30 Jun 2022 15:20:09 +0200 Subject: [PATCH 25/28] model and session are prepopulated --- ckanext/xloader/jobs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 154f0f08..7ead8b0f 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -474,10 +474,12 @@ def update_resource(resource, patch_only=False): or patch the given CKAN resource for file hash """ action = 'resource_update' if not patch_only else 'resource_patch' - from ckan import model - user = get_action('get_site_user')({'model': model, 'ignore_auth': True}, {}) - context = {'model': model, 'session': model.Session, 'ignore_auth': True, - 'user': user['name'], 'auth_user_obj': None} + user = get_action('get_site_user')({'ignore_auth': True}, {}) + context = { + 'ignore_auth': True, + 'user': user['name'], + 'auth_user_obj': None + } get_action(action)(context, resource) From 8b6fa358e7455f7b6099d9b4b26aa41a2a8f7916 Mon Sep 17 00:00:00 2001 From: pdelboca Date: Thu, 30 Jun 2022 15:22:28 +0200 Subject: [PATCH 26/28] Fix import errors --- ckanext/xloader/jobs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 7ead8b0f..052db227 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -19,7 +19,6 @@ import ckan.model as model from ckan.plugins.toolkit import get_action, asbool, ObjectNotFound, config, check_ckan_version import ckan.lib.search as search -from ckan.lib.api_token import get_user_from_token from . import loader from . import db @@ -493,10 +492,12 @@ def _get_user_from_key(api_key_or_token): user = None if check_ckan_version(min_version="2.10"): + from ckan.lib.api_token import get_user_from_token user = get_user_from_token(api_key_or_token) return user if check_ckan_version(min_version="2.9"): + from ckan.lib.api_token import get_user_from_token user = get_user_from_token(api_key_or_token) if not user: user = model.Session.query(model.User).filter_by( From 6f3212fcc0a9df3ceac9df38d1c3dae0b27b004b Mon Sep 17 00:00:00 2001 From: pdelboca Date: Fri, 1 Jul 2022 09:28:13 +0200 Subject: [PATCH 27/28] Add token config to README.md --- README.rst | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 0cb136e6..97681257 100644 --- a/README.rst +++ b/README.rst @@ -111,19 +111,15 @@ offers to contribute this are welcomed. Requirements ------------ -Works with CKAN 2.7.x and later. - -Works with CKAN 2.3.x - 2.6.x if you install ckanext-rq. - Compatibility with core CKAN versions: =============== ============= CKAN version Compatibility =============== ============= -2.3 yes, but no longer tested and you must install ckanext-rq -2.4 yes, but no longer tested and you must install ckanext-rq -2.5 yes, but no longer tested and you must install ckanext-rq -2.6 yes, but no longer tested and you must install ckanext-rq +2.3 no longer tested and you must install ckanext-rq +2.4 no longer tested and you must install ckanext-rq +2.5 no longer tested and you must install ckanext-rq +2.6 no longer tested and you must install ckanext-rq 2.7 yes 2.8 yes 2.9 yes (both Python2 and Python3) @@ -174,7 +170,12 @@ To install XLoader: Ensure ``datastore`` is also listed, to enable CKAN DataStore. -6. If it is a production server, you'll want to store jobs info in a more +6. Starting CKAN 2.10 you will need to set an API Token to be able to + exeute jobs against the server:: + + ckanext.xloader.api_token = + +7. If it is a production server, you'll want to store jobs info in a more robust database than the default sqlite file. It can happily use the main CKAN postgres db by adding this line to the config, but with the same value as you have for ``sqlalchemy.url``:: @@ -183,11 +184,11 @@ To install XLoader: (This step can be skipped when just developing or testing.) -7. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu:: +8. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu:: sudo service apache2 reload -8. Run the worker. First test it on the command-line:: +9. Run the worker. First test it on the command-line:: paster --plugin=ckan jobs -c /etc/ckan/default/ckan.ini worker From 229c61370ee62bf26bb9170f71b7f21d207a4f5c Mon Sep 17 00:00:00 2001 From: pdelboca Date: Fri, 1 Jul 2022 09:35:28 +0200 Subject: [PATCH 28/28] Simplify logic to get user from API Key --- ckanext/xloader/jobs.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ckanext/xloader/jobs.py b/ckanext/xloader/jobs.py index 052db227..2b53cb7f 100644 --- a/ckanext/xloader/jobs.py +++ b/ckanext/xloader/jobs.py @@ -24,6 +24,11 @@ from . import db from .job_exceptions import JobError, HTTPError, DataTooBigError, FileCouldNotBeLoadedError +try: + from ckan.lib.api_token import get_user_from_token +except ImportError: + get_user_from_token = None + SSL_VERIFY = asbool(config.get('ckanext.xloader.ssl_verify', True)) if not SSL_VERIFY: requests.packages.urllib3.disable_warnings() @@ -490,27 +495,12 @@ def _get_user_from_key(api_key_or_token): only API Keys. """ user = None - - if check_ckan_version(min_version="2.10"): - from ckan.lib.api_token import get_user_from_token - user = get_user_from_token(api_key_or_token) - return user - - if check_ckan_version(min_version="2.9"): - from ckan.lib.api_token import get_user_from_token + if get_user_from_token: user = get_user_from_token(api_key_or_token) - if not user: - user = model.Session.query(model.User).filter_by( - apikey=api_key_or_token - ).first() - return user - - if check_ckan_version(min_version="2.7"): + if not user: user = model.Session.query(model.User).filter_by( apikey=api_key_or_token ).first() - return user - return user