Improve Admin CLI

oio/account/rebuilder.py

@@ -18,7 +18,6 @@
 
 from oio import ObjectStorageApi
 from oio.account.client import AccountClient
-from oio.common.exceptions import NotFound, NoSuchContainer
 from oio.common.green import eventlet
 from oio.common.tool import Tool, ToolWorker
 
@@ -53,10 +52,14 @@ def _fetch_items_from_accounts(self):
             yield item
 
     def _fetch_items_from_all_accounts(self):
-        accounts = self.account_client.account_list()
-        for account in accounts:
-            item = self.namespace, account, None
-            yield item
+        try:
+            accounts = self.account_client.account_list()
+            for account in accounts:
+                item = self.namespace, account, None
+                yield item
+        except Exception as exc:
+            self.success = False
+            self.logger.error('Failed to list accounts: %s', exc)
 
     def _fetch_items(self):
         if self.accounts:
@@ -71,11 +74,18 @@ def _fetch_items(self):
             if not self._accounts_to_refresh:
                 break
             item = self._accounts_refreshed.get()
-            self._accounts_to_refresh.remove(item)
-            namespace, account, _ = item
-            containers = self.account_client.container_list(account)
-            for container in containers["listing"]:
-                yield namespace, account, container[0]
+            namespace, account, error = item
+            self._accounts_to_refresh.remove((namespace, account, None))
+            if error:
+                continue
+            try:
+                containers = self.account_client.container_list(account)
+                for container in containers["listing"]:
+                    yield namespace, account, container[0]
+            except Exception as exc:
+                self.success = False
+                self.logger.error('Failed to list containers (account=%s): %s',
+                                  account, exc)
 
     def _get_report(self, status, end_time, counters):
         entries_processed, total_entries_processed, \
@@ -125,10 +135,9 @@ def run(self):
         tasks_res = super(AccountRebuilder, self).run()
         for task_res in tasks_res:
             item, _, error = task_res
-            if error is None:
-                _, _, container = item
-                if container is None:
-                    self._accounts_refreshed.put(item)
+            namespace, account, container = item
+            if container is None:
+                self._accounts_refreshed.put((namespace, account, error))
             yield task_res
 
 
@@ -147,14 +156,6 @@ def _process_item(self, item):
                 namespace, self.tool.namespace))
 
         if container is None:
-            try:
-                self.api.account.account_refresh(account)
-            except NotFound:
-                # account remove in the meantime
-                pass
+            self.api.account.account_refresh(account)
         else:
-            try:
-                self.api.container_refresh(account, container)
-            except NoSuchContainer:
-                # container remove in the meantime
-                pass
+            self.api.container_refresh(account, container)

oio/cli/admin/__init__.py

@@ -15,26 +15,20 @@
 
 """Command-line interface for OpenIO SDS cluster administration."""
 
-import sys
 
-from cliff.app import App
-from cliff.commandmanager import CommandManager
+import sys
 
-from oio import __version__ as oio_version
-from oio.cli import add_common_parser_options
 from oio.cli.common.clientmanager import ClientManager
+from oio.cli.common.shell import CommonShell
 from oio.common.utils import request_id
 
 
-class OpenioAdminApp(App):
+class OpenioAdminApp(CommonShell):
+
+    DEFAULT_VERBOSE_LEVEL = 2
 
     def __init__(self):
-        super(OpenioAdminApp, self).__init__(
-            description=__doc__.strip() if __doc__ else None,
-            version=oio_version,
-            command_manager=CommandManager('openio.admin'),
-            deferred_help=True)
-        self.client_manager = None
+        super(OpenioAdminApp, self).__init__('openio.admin')
 
     def initialize_app(self, argv):
         super(OpenioAdminApp, self).initialize_app(argv)
@@ -48,18 +42,6 @@ def initialize_app(self, argv):
         }
         self.client_manager = ClientManager(options)
 
-    def build_option_parser(self, description, version):
-        parser = super(OpenioAdminApp, self).build_option_parser(
-            description, version)
-        add_common_parser_options(parser)
-        return parser
-
-    def prepare_to_run_command(self, cmd):
-        pass
-
-    def clean_up(self, cmd, result, err):
-        pass
-
     def request_id(self, prefix='ACLI-'):
         """
         Get an ID for requests generated by this application.

oio/cli/admin/common.py

@@ -14,7 +14,19 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-class AccountCommandMixin(object):
+from oio.common.utils import cid_from_name
+
+
+class CommandMixin(object):
+
+    def patch_parser(self, parser):
+        raise NotImplementedError()
+
+    def check_and_load_parsed_args(self, app, parsed_args):
+        raise NotImplementedError()
+
+
+class AccountCommandMixin(CommandMixin):
     """
     Add account-related argmuments to a cliff command.
     """
@@ -27,8 +39,12 @@ def patch_parser(self, parser):
             help='Name of the account to work on.'
         )
 
+    def check_and_load_parsed_args(self, app, parsed_args):
+        if not parsed_args.accounts:
+            parsed_args.accounts = [app.options.account]
 
-class ContainerCommandMixin(object):
+
+class ContainerCommandMixin(CommandMixin):
     """
     Add container-related argmuments to a cliff command.
     """
@@ -47,8 +63,35 @@ def patch_parser(self, parser):
             help="Interpret <container_name> as a container ID",
         )
 
+    def check_and_load_parsed_args(self, app, parsed_args):
+        pass
+
+    def resolve_containers(self, app, parsed_args, no_name=False, no_id=False):
+        containers = list()
+        if parsed_args.is_cid:
+            for container_id in parsed_args.containers:
+                account = None
+                container_name = None
+                if not no_name:
+                    account, container_name = \
+                        app.client_manager.storage.resolve_cid(container_id)
+                if no_id:
+                    container_id = None
+                containers.append((account, container_name, container_id))
+        else:
+            for container_name in parsed_args.containers:
+                account = app.options.account
+                container_id = None
+                if not no_id:
+                    container_id = cid_from_name(account, container_name)
+                if no_name:
+                    account = None
+                    container_name = None
+                containers.append((account, container_name, container_id))
+        return containers
 
-class ObjectCommandMixin(object):
+
+class ObjectCommandMixin(CommandMixin):
     """
     Add object-related argmuments to a cliff command.
     """
@@ -92,7 +135,7 @@ def patch_parser(self, parser):
         )
 
     # TODO(FVE): merge with oio.cli.object.object.ContainerCommandMixin
-    def take_action(self, parsed_args):
+    def check_and_load_parsed_args(self, app, parsed_args):
         if not parsed_args.container and not parsed_args.auto:
             from argparse import ArgumentError
             raise ArgumentError(parsed_args.container,
@@ -101,9 +144,109 @@ def take_action(self, parsed_args):
         # the first object name is in the container variable.
         if parsed_args.auto:
             parsed_args.objects.append(parsed_args.container)
-        if parsed_args.flat_bits:
-            self.app.client_manager.flatns_set_bits(parsed_args.flat_bits)
+            parsed_args.container = None
         if not parsed_args.objects:
             from argparse import ArgumentError
-            raise ArgumentError(None,
-                                "Missing value for object_name")
+            raise ArgumentError(None, 'Missing value for object_name')
+        if parsed_args.flat_bits:
+            app.client_manager.flatns_set_bits(parsed_args.flat_bits)
+
+    def resolve_objects(self, app, parsed_args):
+        containers = set()
+        objects = list()
+        if parsed_args.auto:
+            autocontainer = app.client_manager.flatns_manager
+            for obj in parsed_args.objects:
+                ct = autocontainer(obj)
+                containers.add(ct)
+                objects.append((ct, obj, parsed_args.object_version))
+        else:
+            if parsed_args.is_cid:
+                account, container = \
+                    app.client_manager.storage.resolve_cid(
+                        parsed_args.container)
+            else:
+                account = app.options.account
+                container = parsed_args.container
+            containers.add(container)
+            for obj in parsed_args.objects:
+                objects.append(
+                    (container, obj, parsed_args.object_version))
+        return account, containers, objects
+
+
+class ChunkCommandMixin(CommandMixin):
+    """
+    Add chunk-related argmuments to a cliff command.
+    """
+
+    def patch_parser(self, parser):
+        parser.add_argument(
+            'chunks',
+            metavar='<chunk_url>',
+            nargs='+',
+            help='URL of the chunk to work on.'
+        )
+
+    def check_and_load_parsed_args(self, app, parsed_args):
+        pass
+
+
+class SingleServiceCommandMixin(CommandMixin):
+    """
+    Add service-related argmuments to a cliff command.
+    """
+
+    def patch_parser(self, parser):
+        parser.add_argument(
+            'service',
+            metavar='<service_id>',
+            help=("ID of the service to work on."),
+        )
+
+    def check_and_load_parsed_args(self, app, parsed_args):
+        pass
+
+
+class MultipleServicesCommandMixin(CommandMixin):
+    """
+    Add service-related argmuments to a cliff command.
+    """
+
+    service_type = None
+
+    def patch_parser(self, parser):
+        parser.add_argument(
+            'services',
+            nargs='*',
+            metavar='<service_id>',
+            help=("ID of the service to work on. "
+                  "If no service is specified, work on all."),
+        )
+
+    def check_and_load_parsed_args(self, app, parsed_args):
+        """
+        Load IDs of services.
+        """
+        if not parsed_args.services:
+            parsed_args.services = [
+                s['id'] for s in app.client_manager.conscience.all_services(
+                    self.service_type)]
+
+
+class ProxyCommandMixin(CommandMixin):
+    """
+    Add proxy-related argmuments to a cliff command.
+    """
+
+    def patch_parser(self, parser):
+        parser.add_argument(
+            'service',
+            metavar='<service_id>',
+            nargs='?',
+            help=("ID of the proxy to work on. "
+                  "If not specified, use the local one."),
+        )
+
+    def check_and_load_parsed_args(self, app, parsed_args):
+        pass