From 405d7d83ef8300bc115a27dfc9ac1d3ab3994636 Mon Sep 17 00:00:00 2001 From: kamel-rahim Date: Wed, 10 Apr 2019 11:22:52 +0200 Subject: [PATCH] CLI: list containers from rawx and meta2 --- oio/cli/admin/service_list.py | 178 ++++++++++++++++++++++++++++++++++ setup.cfg | 2 + 2 files changed, 180 insertions(+) create mode 100644 oio/cli/admin/service_list.py diff --git a/oio/cli/admin/service_list.py b/oio/cli/admin/service_list.py new file mode 100644 index 0000000000..26e196cdb6 --- /dev/null +++ b/oio/cli/admin/service_list.py @@ -0,0 +1,178 @@ +# Copyright (C) 2019 OpenIO SAS, as part of OpenIO SDS +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from cliff import lister +from oio.common.exceptions import ClientException + + +class ServiceListCommand(lister.Lister): + """ + A command to display items of a specific service + """ + + reqid_prefix = 'ACLI-LST' + + def __init__(self, *args, **kwargs): + super(ServiceListCommand, self).__init__(*args, **kwargs) + self._cids_cache = {} + + # Cliff ########################################################### + + def get_parser(self, prog_name): + parser = super(ServiceListCommand, self).get_parser(prog_name) + return parser + + # Accessors ####################################################### + + @property + def rdir(self): + """Get an instance of RdirClient.""" + return self.app.client_manager.rdir + + @property + def dir(self): + """Get an instance of DirectoryClient.""" + return self.storage.directory + + @property + def logger(self): + return self.app.client_manager.logger + + @property + def storage(self): + """Get an instance of ObjectStorageApi.""" + return self.app.client_manager.storage + + # Utility ######################################################### + + def translate_cid(self, cid): + """Resolve a CID into account/container names.""" + try: + if cid not in self._cids_cache: + md = self.dir.show(cid=cid) + self._cids_cache[cid] = '/'.join([md.get('account'), + md.get('name')]) + return self._cids_cache[cid] + except ClientException: + pass + return cid + + +class RawxListContainer(ServiceListCommand): + """ + Get the names of all containers which some chunks are stored in the + specified rawx + """ + reqid_prefix = 'ACLI-RLC' + + def _list_containers(self, rawx): + info = self.rdir.status(rawx) + containers = info.get('container') + result = {self.translate_cid(cid=k): v['total'] + for k, v in containers.iteritems()} + result['Total'] = info.get('chunk')['total'] + return result.iteritems() + + def get_parser(self, prog_name): + parser = super(RawxListContainer, self).get_parser(prog_name) + parser.add_argument( + 'rawx_id', + metavar='', + help='ID of the rawx service', + ) + parser.add_argument( + '--limit', + metavar='', + type=int, + default=1000, + help='Limit the number of objects returned (1000 by default)' + ) + return parser + + def take_action(self, parsed_args): + super(RawxListContainer, self).take_action(parsed_args) + return ('Name', 'Chunks'), self._list_containers(parsed_args.rawx_id) + + +class Meta2ListContainer(ServiceListCommand): + """ + Get the names of all containers which are in the specified meta2 + """ + reqid_prefix = 'ACLI-M2LC' + + def get_parser(self, prog_name): + from oio.cli.common.utils import ValueFormatStoreTrueAction + parser = super(Meta2ListContainer, self).get_parser(prog_name) + parser.add_argument( + 'meta2_id', + metavar='', + help='ID of the meta2 service' + ) + parser.add_argument( + '--no-paging', + dest='no_paging', + default=False, + help=("List all elements without paging " + "(and set output format to 'value')"), + action=ValueFormatStoreTrueAction, + ) + parser.add_argument( + '--prefix', + metavar='', + help='Filter list using ' + ) + parser.add_argument( + '--marker', + metavar='', + help='Marker for paging' + ) + parser.add_argument( + '--limit', + metavar='', + type=int, + default=1000, + help='Limit the number of objects returned (1000 by default)' + ) + return parser + + def _list_all_containers(self, meta2, prefix=None): + for r in self.rdir.meta2_index_fetch_all(meta2, prefix=prefix): + yield r['container_url'] + + def _list_containers(self, meta2, **kwargs): + resp = self.rdir.meta2_index_fetch(meta2, **kwargs) + if resp.get('truncated'): + self.log.info( + 'Object listing has been truncated, next marker: %s', + resp.get('next_marker')) + for r in resp.get('records'): + yield r['container_url'] + + def take_action(self, parsed_args): + super(Meta2ListContainer, self).take_action(parsed_args) + kwargs = {} + if parsed_args.marker: + kwargs['marker'] = parsed_args.marker + if parsed_args.prefix: + kwargs['prefix'] = parsed_args.limit + if parsed_args.limit: + kwargs['limit'] = parsed_args.limit + + if parsed_args.no_paging: + containers = self._list_all_containers(parsed_args.meta2_id, + prefix=parsed_args.prefix) + else: + containers = self._list_containers(parsed_args.meta2_id, **kwargs) + return ('Name',), ((v,) for v in containers) diff --git a/setup.cfg b/setup.cfg index e05519b36f..ace11c1e00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -156,6 +156,8 @@ openio.admin = object_check = oio.cli.admin.item_check:ObjectCheck object_locate = oio.cli.admin.item_locate:ObjectLocate proxy_decache = oio.cli.admin.service_decache:ProxyDecache + rawx_list_container = oio.cli.admin.service_list:RawxListContainer + meta2_list_container = oio.cli.admin.service_list:Meta2ListContainer oio.conscience.checker = asn1 = oio.conscience.checker.asn1:Asn1PingChecker