From 30aace46414bbeef02beb75b7128f48fada82150 Mon Sep 17 00:00:00 2001 From: wuzewu Date: Tue, 13 Apr 2021 19:56:28 +0800 Subject: [PATCH] Add `ignore_env_mismatch` in hub.Module. --- paddlehub/commands/install.py | 13 +++++++++++- paddlehub/module/manager.py | 39 ++++++++++++++++++++++++++--------- paddlehub/module/module.py | 34 +++++++++++++++++++++--------- 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/paddlehub/commands/install.py b/paddlehub/commands/install.py index a775ce74e..d79d4c8de 100644 --- a/paddlehub/commands/install.py +++ b/paddlehub/commands/install.py @@ -25,11 +25,22 @@ @register(name='hub.install', description='Install PaddleHub module.') class InstallCommand: + def __init__(self): + self.parser = argparse.ArgumentParser(prog='hub install', add_help=True) + self.parser.add_argument( + '--ignore_env_mismatch', + action='store_true', + help='Whether to ignore the environment mismatch when installing the Module.') + def execute(self, argv: List) -> bool: if not argv: print("ERROR: You must give at least one module to install.") return False + options = [arg for arg in argv if arg.startswith('-')] + argv = [arg for arg in argv if not arg.startswith('-')] + args = self.parser.parse_args(options) + manager = LocalModuleManager() for _arg in argv: if os.path.exists(_arg) and os.path.isdir(_arg): @@ -41,5 +52,5 @@ def execute(self, argv: List) -> bool: name = _arg[0] version = None if len(_arg) == 1 else _arg[1] CacheUpdater("hub_install", name, version).start() - manager.install(name=name, version=version) + manager.install(name=name, version=version, ignore_env_mismatch=args.ignore_env_mismatch) return True diff --git a/paddlehub/module/manager.py b/paddlehub/module/manager.py index 5ab3e7bee..cea5a0d68 100644 --- a/paddlehub/module/manager.py +++ b/paddlehub/module/manager.py @@ -155,19 +155,21 @@ def install(self, version: str = None, source: str = None, update: bool = False, - branch: str = None) -> HubModule: + branch: str = None, + ignore_env_mismatch: bool = False) -> HubModule: ''' Install a HubModule from name or directory or archive file or url. When installing with the name parameter, if a module that meets the conditions (both name and version) already installed, the installation step will be skipped. When installing with other parameter, The locally installed modules will be uninstalled. Args: - name (str|optional): module name to install - directory (str|optional): directory containing module code - archive (str|optional): archive file containing module code - url (str|optional): url points to a archive file containing module code - version (str|optional): module version, use with name parameter - source (str|optional): source containing module code, use with name paramete + name (str|optional): module name to install + directory (str|optional): directory containing module code + archive (str|optional): archive file containing module code + url (str|optional): url points to a archive file containing module code + version (str|optional): module version, use with name parameter + source (str|optional): source containing module code, use with name paramete + ignore_env_mismatch (str|optional): Whether to ignore the environment mismatch when installing the Module. ''' if name: @@ -185,7 +187,7 @@ def install(self, return hub_module_cls if source: return self._install_from_source(name, version, source, update, branch) - return self._install_from_name(name, version) + return self._install_from_name(name, version, ignore_env_mismatch) elif directory: return self._install_from_directory(directory) elif archive: @@ -255,7 +257,7 @@ def _install_from_url(self, url: str) -> HubModule: return self._install_from_archive(file) - def _install_from_name(self, name: str, version: str = None) -> HubModule: + def _install_from_name(self, name: str, version: str = None, ignore_env_mismatch: bool = False) -> HubModule: '''Install HubModule by name search result''' result = module_server.search_module(name=name, version=version) for item in result: @@ -277,7 +279,24 @@ def _install_from_name(self, name: str, version: str = None) -> HubModule: # Cannot find a HubModule that meets the version if valid_infos: - raise EnvironmentMismatchError(name=name, info=valid_infos, version=version) + if not ignore_env_mismatch: + raise EnvironmentMismatchError(name=name, info=valid_infos, version=version) + + # If `ignore_env_mismatch` is set, ignore the problem of environmental mismatch, such as PaddlePaddle or PaddleHub + # version incompatibility. This may cause some unexpected problems during installation or running, but it is useful + # in some cases, for example, the development version of PaddlePaddle(with version number `0.0.0`) is installed + # locally. + if version: + if version in valid_infos: + url = valid_infos[version]['url'] + else: + raise HubModuleNotFoundError(name=name, info=module_infos, version=version) + else: + # Get the maximum version number. + version = sorted([utils.Version(_v) for _v in valid_infos.keys()])[-1] + url = valid_infos[str(version)]['url'] + log.logger.warning('Ignore environmental mismatch of The Module {}-{}'.format(name, version)) + return self._install_from_url(url) raise HubModuleNotFoundError(name=name, info=module_infos, version=version) def _install_from_source(self, name: str, version: str, source: str, update: bool = False, diff --git a/paddlehub/module/module.py b/paddlehub/module/module.py index 6904ed9cc..c0d5451b8 100644 --- a/paddlehub/module/module.py +++ b/paddlehub/module/module.py @@ -359,16 +359,16 @@ class Module(object): name(str): Module name. directory(str|optional): Directory of the module to be loaded, only takes effect when the `name` is not specified. version(str|optional): The version limit of the module, only takes effect when the `name` is specified. When the local - Module does not meet the specified version conditions, PaddleHub will re-request the server to - download the appropriate Module. Default to None, This means that the local Module will be used. - If the Module does not exist, PaddleHub will download the latest version available from the - server according to the usage environment. + Module does not meet the specified version conditions, PaddleHub will re-request the server to download the + appropriate Module. Default to None, This means that the local Module will be used. If the Module does not exist, + PaddleHub will download the latest version available from the server according to the usage environment. source(str|optional): Url of a git repository. If this parameter is specified, PaddleHub will no longer download the - specified Module from the default server, but will look for it in the specified repository. - Default to None. - update(bool|optional): Whether to update the locally cached git repository, only takes effect when the `source` - is specified. Default to False. + specified Module from the default server, but will look for it in the specified repository. Default to None. + update(bool|optional): Whether to update the locally cached git repository, only takes effect when the `source` is + specified. Default to False. branch(str|optional): The branch of the specified git repository. Default to None. + ignore_env_mismatch(bool|optional): Whether to ignore the environment mismatch when installing the Module. Default to + False. ''' def __new__(cls, @@ -379,13 +379,20 @@ def __new__(cls, source: str = None, update: bool = False, branch: str = None, + ignore_env_mismatch: bool = False, **kwargs): if cls.__name__ == 'Module': from paddlehub.server.server import CacheUpdater # This branch come from hub.Module(name='xxx') or hub.Module(directory='xxx') if name: module = cls.init_with_name( - name=name, version=version, source=source, update=update, branch=branch, **kwargs) + name=name, + version=version, + source=source, + update=update, + branch=branch, + ignore_env_mismatch=ignore_env_mismatch, + **kwargs) CacheUpdater("update_cache", module=name, version=version).start() elif directory: module = cls.init_with_directory(directory=directory, **kwargs) @@ -470,13 +477,20 @@ def init_with_name(cls, source: str = None, update: bool = False, branch: str = None, + ignore_env_mismatch: bool = False, **kwargs) -> Union[RunModule, ModuleV1]: '''Initialize Module according to the specified name.''' from paddlehub.module.manager import LocalModuleManager manager = LocalModuleManager() user_module_cls = manager.search(name, source=source, branch=branch) if not user_module_cls or not user_module_cls.version.match(version): - user_module_cls = manager.install(name=name, version=version, source=source, update=update, branch=branch) + user_module_cls = manager.install( + name=name, + version=version, + source=source, + update=update, + branch=branch, + ignore_env_mismatch=ignore_env_mismatch) directory = manager._get_normalized_path(user_module_cls.name)