Source code for kamaki.cli.cmds.pithos

# Copyright 2011-2015 GRNET S.A. All rights reserved.
#
# Redistribution and use in source and binary forms, with or
# without modification, are permitted provided that the following
# conditions are met:
#
#   1. Redistributions of source code must retain the above
#      copyright notice, this list of conditions and the following
#      disclaimer.
#
#   2. Redistributions in binary form must reproduce the above
#      copyright notice, this list of conditions and the following
#      disclaimer in the documentation and/or other materials
#      provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# The views and conclusions contained in the software and
# documentation are those of the authors and should not be
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.command

from time import localtime, strftime
from io import StringIO
from pydoc import pager
from os import path, walk, makedirs
from threading import activeCount, enumerate as activethreads

from kamaki.clients.pithos import PithosClient, ClientError
from kamaki.clients.utils import escape_ctrl_chars

from kamaki.cli import command
from kamaki.cli.cmdtree import CommandTree
from kamaki.cli.cmds import (
    CommandInit, dont_raise, OptionalOutput, NameFilter, errors, client_log)
from kamaki.cli.errors import (
    CLIBaseUrlError, CLIError, CLIInvalidArgument, raiseCLIError,
    CLISyntaxError)
from kamaki.cli.argument import (
    FlagArgument, IntArgument, ValueArgument, DateArgument, KeyValueArgument,
    ProgressBarArgument, RepeatableArgument, DataSizeArgument,
    UserAccountArgument)
from kamaki.cli.utils import format_size, get_path_size, guess_mime_type, bold

file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
container_cmds = CommandTree(
    'container', 'Pithos+/Storage container level API commands')
sharer_cmds = CommandTree('sharer', 'Pithos+/Storage sharers')
group_cmds = CommandTree('group', 'Pithos+/Storage user groups')
namespaces = [file_cmds, container_cmds, sharer_cmds, group_cmds]


class _PithosInit(CommandInit):
    """Initilize a pithos+ client
    There is always a default account (current user uuid)
    There is always a default container (pithos)
    """

    @dont_raise(KeyError)
    def _custom_container(self):
        return self.config.get_cloud(self.cloud, 'pithos_container')

    @dont_raise(KeyError)
    def _custom_uuid(self):
        return self.config.get_cloud(self.cloud, 'pithos_uuid')

    def _set_account(self):
        self.account = self._custom_uuid()
        if self.account:
            return
        astakos = getattr(self, 'astakos', None)
        if astakos:
            self.account = astakos.user_term('id', self.token)
        else:
            raise CLIBaseUrlError(service='astakos')

    @errors.Generic.all
    @client_log
    def _run(self):
        self.client = self.get_client(PithosClient, 'pithos')
        self.endpoint_url = self.client.endpoint_url
        self.token = self.client.token
        self._set_account()
        self.client.account = self.account
        self.container = self._custom_container() or 'pithos'
        self.client.container = self.container

    def main(self):
        self._run()


class _PithosAccount(_PithosInit):
    """Setup account"""

    def __init__(self, arguments={}, astakos=None, cloud=None):
        super(_PithosAccount, self).__init__(arguments, astakos, cloud)
        self['account'] = UserAccountArgument(
            'A user UUID or name', ('-A', '--account'))
        self.arguments['account'].account_client = astakos

    def print_objects(self, object_list):
        for index, obj in enumerate(object_list):
            pretty_obj = obj.copy()
            index += 1
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
            if 'subdir' in obj:
                continue
            if self.object_is_dir(obj):
                size = 'D'
            else:
                size = format_size(obj['bytes'])
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
            oname = escape_ctrl_chars(obj['name'])
            oname = oname if self['more'] else bold(oname)
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
            if self['detail']:
                self.writeln('%s%s' % (prfx, oname))
                self.print_dict(pretty_obj, exclude=('name'))
                self.writeln()
            else:
                oname = '%s%9s %s' % (prfx, size, oname)
                oname += '/' if self.object_is_dir(obj) else u''
                self.writeln(oname)

    @staticmethod
    def object_is_dir(remote_dict):
        """:returns: True if the content type of the object is
            'applcation/directory' or 'application/folder'
        """
        content_type = remote_dict.get(
            'content_type', remote_dict.get('content-type', ''))
        dir_types = ['application/directory', 'application/folder']
        return any([t in content_type for t in dir_types])

    def _run(self):
        super(_PithosAccount, self)._run()
        self.client.account = self['account'] or getattr(
            self, 'account', getattr(self.client, 'account', None))


class _PithosContainer(_PithosAccount):
    """Setup container"""

    def __init__(self, arguments={}, astakos=None, cloud=None):
        super(_PithosContainer, self).__init__(arguments, astakos, cloud)
        self['container'] = ValueArgument(
            'Use this container (default: pithos)', ('-C', '--container'))

    @staticmethod
    def resolve_pithos_url(url):
        """Match urls of one of the following formats:
        pithos://ACCOUNT/CONTAINER/OBJECT_PATH
        /CONTAINER/OBJECT_PATH
        return account, container, path
        """
        account, container, obj_path, prefix = '', '', url, 'pithos://'
        if url.startswith(prefix):
            account, sep, url = url[len(prefix):].partition('/')
            url = '/%s' % url
        if url.startswith('/'):
            container, sep, obj_path = url[1:].partition('/')
        return account, container, obj_path

    @errors.Pithos.container
    def _container_exists(self, container=None):
        bu_cont = self.container
        container = container or self.container
        try:
            self.client.get_container_info(container)
        finally:
            self.container = bu_cont

    def _run(self, url=None):
        acc, con, self.path = self.resolve_pithos_url(url or '')
        super(_PithosContainer, self)._run()
        self.container = con or self['container'] or getattr(
            self, 'container', None) or getattr(self.client, 'container', '')
        self.client.account = acc or self.client.account
        self.client.container = self.container


@command(file_cmds)
[docs]class file_info(_PithosContainer, OptionalOutput): """Get information/details about a file""" arguments = dict( object_version=ValueArgument( 'download a file of a specific version', '--object-version'), hashmap=FlagArgument( 'Get file hashmap instead of details', '--hashmap'), matching_etag=ValueArgument( 'show output if ETags match', '--if-match'), non_matching_etag=ValueArgument( 'show output if ETags DO NOT match', '--if-none-match'), modified_since_date=DateArgument( 'show output modified since then', '--if-modified-since'), unmodified_since_date=DateArgument( 'show output unmodified since then', '--if-unmodified-since'), sharing=FlagArgument( 'show object permissions and sharing information', '--sharing'), metadata=FlagArgument('show only object metadata', '--metadata'), versions=FlagArgument( 'show the list of versions for the file', '--object-versions') )
[docs] def version_print(self, versions): return {'/%s/%s' % (self.container, self.path): [ dict(version_id=vitem[0], created=strftime( '%d-%m-%Y %H:%M:%S', localtime(float(vitem[1])))) for vitem in versions]}
@errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): try: if self['hashmap']: r = self.client.get_object_hashmap( self.path, version=self['object_version'], if_match=self['matching_etag'], if_none_match=self['non_matching_etag'], if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date']) elif self['sharing']: r = self.client.get_object_sharing(self.path) r['public url'] = self.client.get_object_info( self.path, version=self['object_version']).get( 'x-object-public', None) elif self['metadata']: r, preflen = dict(), len('x-object-meta-') for k, v in self.client.get_object_meta(self.path).items(): r[k[preflen:]] = v elif self['versions']: r = self.version_print( self.client.get_object_versionlist(self.path)) else: r = self.client.get_object_info( self.path, version=self['object_version']) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise self.print_(r, self.print_dict)
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) self._run()
@command(file_cmds)
[docs]class file_list(_PithosContainer, OptionalOutput, NameFilter): """List all objects in a container or a directory""" arguments = dict( detail=FlagArgument('detailed output', ('-l', '--list')), limit=IntArgument('limit number of listed items', ('-n', '--number')), marker=ValueArgument('output greater that marker', '--marker'), delimiter=ValueArgument('show output up to delimiter', '--delimiter'), meta=ValueArgument( 'show output with specified meta keys', '--meta', default=[]), if_modified_since=ValueArgument( 'show output modified since then', '--if-modified-since'), if_unmodified_since=ValueArgument( 'show output not modified since then', '--if-unmodified-since'), until=DateArgument('show metadata until then', '--until'), format=ValueArgument( 'format to parse until data (default: d/m/Y H:M:S )', '--format'), shared_by_me=FlagArgument( 'show only files shared to other users', '--shared-by-me'), public=FlagArgument('show only published objects', '--public'), more=FlagArgument('read long results', '--more'), enum=FlagArgument('Enumerate results', '--enumerate'), recursive=FlagArgument( 'Recursively list containers and their contents', ('-r', '--recursive')) ) @errors.Pithos.container def _container_info(self): r = self.client.container_get( limit=False if self['more'] else self['limit'], marker=self['marker'], prefix=self.path, delimiter=self['delimiter'], path=self['name_pref'] or '', show_only_shared=self['shared_by_me'], public=self['public'], if_modified_since=self['if_modified_since'], if_unmodified_since=self['if_unmodified_since'], until=self['until'], meta=self['meta']) files = list(r.json or []) return files @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): r = self._container_info() if not r: if self.path: obj_path = '/%s/%s' % (self.container, self.path) obj_info = self.client.get_object_info(self.path) if self.object_is_dir(obj_info): self.error('Directory %s is empty' % obj_path) else: cnt_msg = '[/%s]' % self.container if ( 'pithos' == self.container) else '/%s' % self.container raise CLIError( 'Object %s is not a directory' % obj_path, importance=2, details=[ 'Use "list" to see contents of containers or ' 'directories', 'To list all objects in container', ' kamaki file list %s' % cnt_msg, 'To list all objects in a directory', ' kamaki file list [/CONTAINER/]DIRECTORY', 'To get details on object', ' kamaki file info %s' % obj_path]) else: self.error('Container "%s" is empty' % self.client.container) files = self._filter_by_name(r) if self['more']: outbu, self._out = self._out, StringIO() try: if self['output_format']: self.print_(files) else: self.print_objects(files) finally: if self['more']: pager(self._out.getvalue()) self._out = outbu
[docs] def main(self, path_or_url=''): super(self.__class__, self)._run(path_or_url) self._run()
def _assert_path(self, path_or_url): if not self.path: raise CLISyntaxError( 'Invalid or incomplete location %s' % path_or_url, details=['Location format', '[[pithos://UUID]/CONTAINER/]PATH']) @command(file_cmds)
[docs]class file_modify(_PithosContainer): """Modify the attributes of a file or directory object""" arguments = dict( uuid_for_read_permission=RepeatableArgument( 'Give read access to user/group (can be repeated, accumulative). ' 'Format for users: UUID . Format for groups: UUID:GROUP . ' 'Use * for all users/groups', '--read-permission'), uuid_for_write_permission=RepeatableArgument( 'Give write access to user/group (can be repeated, accumulative). ' 'Format for users: UUID . Format for groups: UUID:GROUP . ' 'Use * for all users/groups', '--write-permission'), no_permissions=FlagArgument('Remove permissions', '--no-permissions'), metadata_to_set=KeyValueArgument( 'Add metadata (KEY=VALUE) to an object (can be repeated)', '--metadata-add'), metadata_key_to_delete=RepeatableArgument( 'Delete object metadata (can be repeated)', '--metadata-del'), ) required = [ 'uuid_for_read_permission', 'metadata_to_set', 'uuid_for_write_permission', 'no_permissions', 'metadata_key_to_delete'] @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): try: if self['uuid_for_read_permission'] or self[ 'uuid_for_write_permission']: perms = self.client.get_object_sharing(self.path) read, write = perms.get('read', ''), perms.get('write', '') read = read.split(',') if read else [] write = write.split(',') if write else [] read += (self['uuid_for_read_permission'] or []) write += (self['uuid_for_write_permission'] or []) self.client.set_object_sharing( self.path, read_permission=read, write_permission=write) self.print_dict(self.client.get_object_sharing(self.path)) if self['no_permissions']: self.client.del_object_sharing(self.path) metadata = self['metadata_to_set'] or dict() for k in (self['metadata_key_to_delete'] or []): metadata[k] = '' if metadata: self.client.set_object_meta(self.path, metadata) self.print_dict(self.client.get_object_meta(self.path)) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) if self['no_permissions'] and ( self['uuid_for_read_permission'] or self[ 'uuid_for_write_permission']): raise CLIInvalidArgument( '%s cannot be used with other permission arguments' % ( self.arguments['no_permissions'].lvalue)) _assert_path(self, path_or_url) self._run()
@command(file_cmds)
[docs]class file_publish(_PithosContainer): """Publish an object (creates a public URL)""" @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): try: self.writeln(self.client.publish_object(self.path)) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) self._run()
@command(file_cmds)
[docs]class file_unpublish(_PithosContainer): """Unpublish an object""" @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): try: self.client.unpublish_object(self.path) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) self._run()
@command(file_cmds)
[docs]class file_create(_PithosContainer): """Create an empty object""" arguments = dict( content_type=ValueArgument( 'Set content type (default: application/octet-stream)', '--content-type', default='application/octet-stream') ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): self.client.create_object(self.path, self['content_type'])
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) _assert_path(self, path_or_url) self._run()
@command(file_cmds)
[docs]class file_mkdir(_PithosContainer): """Create a directory object Equivalent to kamaki file create --content-type='application/directory' """ @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self, path): self.client.create_directory(self.path)
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) _assert_path(self, path_or_url) self._run(self.path)
@command(file_cmds)
[docs]class file_delete(_PithosContainer): """Delete a file or directory object""" arguments = dict( until_date=DateArgument('remove history until then', '--until'), yes=FlagArgument('Do not prompt for permission', '--yes'), recursive=FlagArgument( 'If a directory, empty first', ('-r', '--recursive')), delimiter=ValueArgument( 'delete objects prefixed with <object><delimiter>', '--delimiter') ) @errors.Pithos.object_path def _delete_object(self): self.client.get_object_info(self.path) if self['yes'] or self.ask_user( 'Delete /%s/%s ?' % (self.container, self.path)): # See if any objects exist under prefix # Add a trailing / to object's name prefix = self.path.rstrip('/') + '/' result = self.client.container_get(prefix=prefix) if result.json: count = len(result.json) self.error(' * %d other object(s) with %s as prefix found' % ( count, prefix)) if self['recursive']: msg = 'The above %d object(s) will be deleted, too' % \ count else: msg = 'The above %d object(s) will be preserved,' \ ' but the directory structure' \ ' will become inconsistent' % count self.error(' * %s!' % msg) if not result.json or self.ask_user("Continue?"): self.client.del_object( self.path, until=self['until_date'], delimiter='/' if self['recursive'] else self['delimiter']) else: self.error('Aborted') @errors.Pithos.container def _empty_container(self): self.client.get_container_info() if self['yes'] or self.ask_user( 'Empty container /%s ?' % self.container): self.client.container_delete(self.container, delimiter='/') else: self.error('Aborted') @errors.Generic.all @errors.Pithos.connection def _run(self): if self.path: self._delete_object() else: self._empty_container()
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) self._run()
class _PithosFromTo(_PithosContainer): sd_arguments = dict( destination_user=UserAccountArgument( 'UUID or username, default: current user', '--to-account'), destination_container=ValueArgument( 'default: pithos', '--to-container'), source_prefix=FlagArgument( 'Transfer all files that are prefixed with SOURCE PATH . If the ' 'destination path is specified, replace SOURCE_PATH with ' 'DESTINATION_PATH', ('-r', '--recursive')), force=FlagArgument( 'Overwrite destination objects, if needed', ('-f', '--force')), source_version=ValueArgument( 'The version of the source object', '--source-version') ) def __init__(self, arguments={}, astakos=None, cloud=None): self.arguments.update(arguments) self.arguments.update(self.sd_arguments) super(_PithosFromTo, self).__init__( self.arguments, astakos, cloud) self.arguments['destination_user'].account_client = self.astakos def _report_transfer(self, src, dst, transfer_name): if not dst: if transfer_name in ('move', ): self.error(' delete source directory %s' % src) return dst_prf = '' if self.account == self.dst_client.account else ( 'pithos://%s' % self.dst_client.account) full_dest_path = '%s/%s/%s' % (dst_prf, self.dst_client.container, dst) if src: src_prf = '' if self.account == self.dst_client.account else ( 'pithos://%s' % self.account) full_src_path = '/%s/%s/%s' % (src_prf, self.container, src) self.error(' %s %s --> %s' % ( transfer_name, full_src_path, full_dest_path)) else: self.error(' mkdir %s' % full_dest_path) @errors.Generic.all @errors.Pithos.account def _src_dst(self, version=None): """Preconditions: self.account, self.container, self.path self.dst_acc, self.dst_con, self.dst_path They should all be configured properly :returns: [(src_path, dst_path), ...], if src_path is None, create destination directory """ src_objects, dst_objects, pairs = dict(), dict(), [] try: for obj in self.dst_client.list_objects( prefix=self.dst_path or self.path or '/'): dst_objects[obj['name']] = obj except ClientError as ce: if ce.status in (404, ): raise CLIError( 'Destination container pithos://%s/%s not found' % ( self.dst_client.account, self.dst_client.container), importance=2) raise ce if self['source_prefix']: # Copy and replace prefixes for src_obj in self.client.list_objects(prefix=self.path): src_objects[src_obj['name']] = src_obj for src_path, src_obj in src_objects.items(): dst_path = '%s%s' % ( self.dst_path or self.path, src_path[len(self.path):]) dst_obj = dst_objects.get(dst_path, None) if self['force'] or not dst_obj: # Just do it pairs.append(( None if self.object_is_dir(src_obj) else src_path, dst_path)) if self.object_is_dir(src_obj): pairs.append((self.path or dst_path, None)) elif not any([ self.object_is_dir(dst_obj), self.object_is_dir(src_obj)]): raise CLIError( 'Destination object exists', importance=2, details=[ 'Failed while transfering:', ' pithos://%s/%s/%s' % ( self.account, self.container, src_path), '--> pithos://%s/%s/%s' % ( self.dst_client.account, self.dst_client.container, dst_path), 'Use %s to transfer overwrite' % ( self.arguments['force'].lvalue)]) else: # One object transfer try: src_version_arg = self.arguments.get('source_version', None) src_obj = self.client.get_object_info( self.path, version=src_version_arg.value if src_version_arg else None) except ClientError as ce: if ce.status in (204, ): raise CLIError( 'Missing specific path container %s' % self.container, importance=2, details=[ 'To transfer container contents %s' % ( self.arguments['source_prefix'].lvalue)]) raise dst_path = self.dst_path or self.path dst_obj = dst_objects.get(dst_path or self.path, None) if self['force'] or not dst_obj: pairs.append(( None if self.object_is_dir(src_obj) else self.path, dst_path)) if self.object_is_dir(src_obj): pairs.append((self.path or dst_path, None)) elif self.object_is_dir(src_obj): raise CLIError( 'Cannot transfer an application/directory object', importance=2, details=[ 'The object pithos://%s/%s/%s is a directory' % ( self.account, self.container, self.path), 'To recursively copy a directory, use', ' %s' % self.arguments['source_prefix'].lvalue, 'To create a file, use', ' /file create (general purpose)', ' /file mkdir (a directory object)']) else: raise CLIError( 'Destination object exists', importance=2, details=[ 'Failed while transfering:', ' pithos://%s/%s/%s' % ( self.account, self.container, self.path), '--> pithos://%s/%s/%s' % ( self.dst_client.account, self.dst_client.container, dst_path), 'Use %s to transfer overwrite' % ( self.arguments['force'].lvalue)]) return pairs def _run(self, source_path_or_url, destination_path_or_url=''): super(_PithosFromTo, self)._run(source_path_or_url) dst_acc, dst_con, dst_path = self.resolve_pithos_url( destination_path_or_url) self.dst_client = PithosClient( endpoint_url=self.client.endpoint_url, token=self.client.token, container=self[ 'destination_container'] or dst_con or self.client.container, account=self['destination_user'] or dst_acc or self.account) self.dst_path = dst_path or self.path @command(file_cmds)
[docs]class file_copy(_PithosFromTo): """Copy objects, even between different accounts or containers""" arguments = dict( public=ValueArgument('publish new object', '--public'), content_type=ValueArgument( 'change object\'s content type', '--content-type'), source_version=ValueArgument( 'The version of the source object', '--object-version') ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container @errors.Pithos.account def _run(self): for src, dst in self._src_dst(self['source_version']): self._report_transfer(src, dst, 'copy') if src and dst: self.dst_client.copy_object( src_container=self.client.container, src_object=src, dst_container=self.dst_client.container, dst_object=dst, source_account=self.client.account, source_version=self['source_version'], public=self['public'], content_type=self['content_type']) elif dst: self.dst_client.create_directory(dst)
[docs] def main(self, source_path_or_url, destination_path_or_url=None): super(file_copy, self)._run( source_path_or_url, destination_path_or_url or '') self._run()
@command(file_cmds)
[docs]class file_move(_PithosFromTo): """Move objects, even between different accounts or containers""" arguments = dict( public=ValueArgument('publish new object', '--public'), content_type=ValueArgument( 'change object\'s content type', '--content-type') ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container @errors.Pithos.account def _run(self): for src, dst in self._src_dst(): self._report_transfer(src, dst, 'move') if src and dst: self.dst_client.move_object( src_container=self.client.container, src_object=src, dst_container=self.dst_client.container, dst_object=dst, source_account=self.account, public=self['public'], content_type=self['content_type']) elif dst: self.dst_client.create_directory(dst) else: self.client.del_object(src)
[docs] def main(self, source_path_or_url, destination_path_or_url=None): super(file_move, self)._run( source_path_or_url, destination_path_or_url or '') self._run()
@command(file_cmds)
[docs]class file_append(_PithosContainer): """Append local file to (existing) remote object The remote object should exist. If the remote object is a directory, it is transformed into a file. In the later case, objects under the directory remain intact. """ arguments = dict( progress_bar=ProgressBarArgument( 'do not show progress bar', ('-N', '--no-progress-bar'), default=False), max_threads=IntArgument('default: 1', '--threads'), ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self, local_path): if self['max_threads'] > 0: self.client.MAX_THREADS = int(self['max_threads']) (progress_bar, upload_cb) = self._safe_progress_bar('Appending') try: with open(local_path, 'rb') as f: self.client.append_object(self.path, f, upload_cb) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise ce finally: self._safe_progress_bar_finish(progress_bar)
[docs] def main(self, local_path, remote_path_or_url): super(self.__class__, self)._run(remote_path_or_url) self._run(local_path)
@command(file_cmds)
[docs]class file_truncate(_PithosContainer): """Truncate remote file up to size""" arguments = dict( size_in_bytes=IntArgument('Length of file after truncation', '--size') ) required = ('size_in_bytes', ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path @errors.Pithos.object_size def _run(self, size): try: self.client.truncate_object(self.path, size) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) self._run(size=self['size_in_bytes'])
@command(file_cmds)
[docs]class file_overwrite(_PithosContainer): """Overwrite part of a remote file""" arguments = dict( progress_bar=ProgressBarArgument( 'do not show progress bar', ('-N', '--no-progress-bar'), default=False), start_position=IntArgument('File position in bytes', '--from'), end_position=IntArgument('File position in bytes', '--to'), object_version=ValueArgument('File to overwrite', '--object-version'), ) required = ('start_position', 'end_position') @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path @errors.Pithos.object_size def _run(self, local_path, start, end): start, end = int(start), int(end) (progress_bar, upload_cb) = self._safe_progress_bar( 'Overwrite %s bytes' % (end - start)) try: with open(path.abspath(local_path), 'rb') as f: self.client.overwrite_object( obj=self.path, start=start, end=end, source_file=f, source_version=self['object_version'], upload_cb=upload_cb) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise finally: self._safe_progress_bar_finish(progress_bar)
[docs] def main(self, local_path, path_or_url): super(self.__class__, self)._run(path_or_url) self.path = self.path or path.basename(local_path) self._run( local_path=local_path, start=self['start_position'], end=self['end_position'])
@command(file_cmds)
[docs]class file_upload(_PithosContainer): """Upload a file The default destination is /pithos/NAME where NAME is the base name of the source path""" arguments = dict( max_threads=IntArgument('default: 5', '--threads'), content_encoding=ValueArgument( 'set MIME content type', '--content-encoding'), content_disposition=ValueArgument( 'specify objects presentation style', '--content-disposition'), content_type=ValueArgument('specify content type', '--content-type'), uuid_for_read_permission=RepeatableArgument( 'Give read access to a user or group (can be repeated) ' 'Use * for all users', '--read-permission'), uuid_for_write_permission=RepeatableArgument( 'Give write access to a user or group (can be repeated) ' 'Use * for all users', '--write-permission'), public=FlagArgument('make object publicly accessible', '--public'), progress_bar=ProgressBarArgument( 'do not show progress bar', ('-N', '--no-progress-bar'), default=False), overwrite=FlagArgument('Force (over)write', ('-f', '--force')), recursive=FlagArgument( 'Recursively upload directory *contents* + subdirectories', ('-r', '--recursive')), unchunked=FlagArgument( 'Upload file as one block (not recommended)', '--unchunked'), md5_checksum=ValueArgument( 'Confirm upload with a custom checksum (MD5)', '--etag'), use_hashes=FlagArgument( 'Source file contains hashmap not data', '--source-is-hashmap'), ) def _sharing(self): sharing = dict() readlist = self['uuid_for_read_permission'] if readlist: sharing['read'] = self['uuid_for_read_permission'] writelist = self['uuid_for_write_permission'] if writelist: sharing['write'] = self['uuid_for_write_permission'] return sharing or None def _check_container_limit(self, path): cl_dict = self.client.get_container_limit() try: container_limit = int(cl_dict['x-container-policy-quota']) except KeyError: container_limit = 0 r = self.client.container_get() used_bytes = sum(int(o['bytes']) for o in r.json) path_size = get_path_size(path) if container_limit and path_size > (container_limit - used_bytes): raise CLIError( 'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % ( self.client.container, format_size(container_limit), format_size(used_bytes), format_size(path_size), path), details=[ 'Check accound limit: /file quota', 'Check container limit:', '\t/file containerlimit get %s' % self.client.container, 'Increase container limit:', '\t/file containerlimit set <new limit> %s' % ( self.client.container)]) def _src_dst(self, local_path, remote_path, objlist=None): lpath = path.abspath(local_path) short_path = path.basename(path.abspath(local_path)) rpath = remote_path or short_path if path.isdir(lpath): if not self['recursive']: raise CLIError('%s is a directory' % lpath, details=[ 'Use %s to upload directories & contents' % ( self.arguments['recursive'].lvalue)]) robj = self.client.container_get(path=rpath) if not self['overwrite']: if robj.json: raise CLIError( 'Objects/files prefixed as %s already exist' % rpath, details=['Existing objects:'] + ['\t/%s\t[%s]' % ( o['name'], o['content_type']) for o in robj.json] + [ 'Use -f to add, overwrite or resume']) else: try: topobj = self.client.get_object_info(rpath) if not self.object_is_dir(topobj): raise CLIError( 'Object /%s/%s exists but not a directory' % ( self.container, rpath), details=['Use -f to overwrite']) except ClientError as ce: if ce.status not in (404, ): raise self._check_container_limit(lpath) prev = '' for top, subdirs, files in walk(lpath): if top != prev: prev = top try: rel_path = rpath + top.split(lpath)[1] except IndexError: rel_path = rpath # Use the '/' separator for directories that # are about to be created in Pithos rel_path = rel_path.replace(path.sep, '/') self.error('remote: mkdir /%s/%s' % ( self.client.container, rel_path)) self.client.create_directory(rel_path) for f in files: fpath = path.join(top, f) if path.isfile(fpath): rel_path = rel_path.replace(path.sep, '/') pathfix = f.replace(path.sep, '/') yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix) else: self.error('%s not a regular file' % fpath) else: if not path.isfile(lpath): raise CLIError(('%s is not a regular file' % lpath) if ( path.exists(lpath)) else '%s does not exist' % lpath) try: robj = self.client.get_object_info(rpath) if remote_path and self.object_is_dir(robj): rpath += '/%s' % (short_path.replace(path.sep, '/')) self.client.get_object_info(rpath) if not self['overwrite']: raise CLIError( 'Object /%s/%s already exists' % ( self.container, rpath), details=['use -f to overwrite / resume']) except ClientError as ce: if ce.status in (404, ): self._container_exists() else: raise self._check_container_limit(lpath) yield open(lpath, 'rb'), rpath def _run(self, local_path, remote_path): self.client.MAX_THREADS = int(self['max_threads'] or 5) params = dict( content_encoding=self['content_encoding'], content_type=self['content_type'], content_disposition=self['content_disposition'], sharing=self._sharing(), public=self['public']) container_info_cache = dict() rpref = ('pithos://%s' % self['account']) if self['account'] else '' for f, rpath in self._src_dst(local_path, remote_path): self.error('%s --> %s/%s/%s' % ( f.name, rpref, self.client.container, rpath)) if not (self['content_type'] and self['content_encoding']): ctype, cenc = guess_mime_type(f.name) params['content_type'] = self['content_type'] or ctype params['content_encoding'] = self['content_encoding'] or cenc if self['unchunked']: self.client.upload_object_unchunked( rpath, f, etag=self['md5_checksum'], withHashFile=self['use_hashes'], **params) else: try: (progress_bar, upload_cb) = self._safe_progress_bar( 'Uploading %s' % f.name.split(path.sep)[-1]) if progress_bar: hash_bar = progress_bar.clone() hash_cb = hash_bar.get_generator( 'Calculating block hashes') else: hash_cb = None caller_id = self.astakos.user_term('id') if self.client.account != caller_id: params['target_account'], self.client.account = ( self.client.account, caller_id) self.client.upload_object( rpath, f, hash_cb=hash_cb, upload_cb=upload_cb, container_info_cache=container_info_cache, **params) except KeyboardInterrupt: timeout = 0.5 msg = '\n' while activeCount() > 1: msg += 'Wait for %s threads: ' % (activeCount() - 1) self._err.write(msg) for thread in activethreads(): try: thread.join(timeout) self._err.write( '.' if thread.isAlive() else '*') self._err.flush() except RuntimeError: continue finally: timeout += 0.1 self._err.flush() msg = '\b' * len(msg) raise CLIError('Upload canceled by user') except Exception: self._safe_progress_bar_finish(progress_bar) raise finally: self._safe_progress_bar_finish(progress_bar) if self['public']: obj = self.client.get_object_info(rpath) self.write('%s\n' % obj.get('x-object-public', '')) self.error('Upload completed')
[docs] def main(self, local_path, remote_path_or_url=None): super(self.__class__, self)._run(remote_path_or_url) if local_path.endswith('.') or local_path.endswith(path.sep): remote_path = self.path or '' else: remote_path = self.path or path.basename(path.abspath(local_path)) self._run(local_path=local_path, remote_path=remote_path)
[docs]class RangeArgument(ValueArgument): """ :value type: string of the form <start>-<end> where <start> and <end> are integers :value returns: the input string, after type checking <start> and <end> """ @property def value(self): return getattr(self, '_value', self.default) @value.setter
[docs] def value(self, newvalues): if newvalues: self._value = getattr(self, '_value', self.default) for newvalue in newvalues.split(','): self._value = ('%s,' % self._value) if self._value else '' start, sep, end = newvalue.partition('-') if sep: if start: start, end = (int(start), int(end)) if start > end: raise CLIInvalidArgument( 'Invalid range %s' % newvalue, details=[ 'Valid range formats', ' START-END', ' UP_TO', ' -FROM', 'where all values are integers', 'OR a compination (csv), e.g.,', ' %s=5,10-20,-5' % self.lvalue]) self._value += '%s-%s' % (start, end) else: self._value += '-%s' % int(end) else: self._value += '%s' % int(start)
@command(file_cmds)
[docs]class file_cat(_PithosContainer): """Fetch remote file contents""" arguments = dict( range=RangeArgument('show range of data e.g., 5,10-20,-5', '--range'), if_match=ValueArgument('show output if ETags match', '--if-match'), if_none_match=ValueArgument( 'show output if ETags match', '--if-none-match'), if_modified_since=DateArgument( 'show output modified since then', '--if-modified-since'), if_unmodified_since=DateArgument( 'show output unmodified since then', '--if-unmodified-since'), object_version=ValueArgument( 'Get contents of the chosen version', '--object-version'), buffer_blocks=IntArgument( 'Size of buffer in blocks (default: 4)', '--buffer-blocks') ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.object_path def _run(self): try: # self.client.download_object( self.client.stream_down( self.path, self._out, range_str=self['range'], version=self['object_version'], if_match=self['if_match'], if_none_match=self['if_none_match'], if_modified_since=self['if_modified_since'], if_unmodified_since=self['if_unmodified_since'], buffer_blocks=self['buffer_blocks']) except ClientError as ce: if ce.status in (404, ): self._container_exists() raise self._out.flush()
[docs] def main(self, path_or_url): super(self.__class__, self)._run(path_or_url) if self['buffer_blocks'] is not None and self['buffer_blocks'] < 1: arg = self.arguments['buffer_blocks'] raise CLIInvalidArgument( 'Invalid value %s' % arg.value, importance=2, details=[ '%s must be a possitive integer' % arg.lvalue]) self._run()
@command(file_cmds)
[docs]class file_download(_PithosContainer): """Download a remote file or directory object to local file system""" arguments = dict( resume=FlagArgument( 'Resume/Overwrite (attempt resume, else overwrite)', ('-f', '--resume')), range=RangeArgument( 'Download only that range of data e.g., 5,10-20,-5', '--range'), matching_etag=ValueArgument('download iff ETag match', '--if-match'), non_matching_etag=ValueArgument( 'download iff ETags DO NOT match', '--if-none-match'), modified_since_date=DateArgument( 'download iff remote file is modified since then', '--if-modified-since'), unmodified_since_date=DateArgument( 'show output iff remote file is unmodified since then', '--if-unmodified-since'), object_version=ValueArgument( 'download a file of a specific version', '--object-version'), max_threads=IntArgument('default: 5', '--threads'), progress_bar=ProgressBarArgument( 'do not show progress bar', ('-N', '--no-progress-bar'), default=False), recursive=FlagArgument( 'Download a remote directory object and its contents', ('-r', '--recursive')) ) def _src_dst(self, local_path): """Create a list of (src, dst) where src is a remote location and dst is an open file descriptor. Directories are denoted as (None, dirpath) and they are pretended to other objects in a very strict order (shorter to longer path).""" ret, obj = [], None # The prefix is actually the relative remote path without # the trailing separator. prefix = self.path.rstrip('/') try: # prefix here is the object's path we requested to download if prefix: obj = self.client.get_object_info( prefix, version=self['object_version']) obj.setdefault('name', prefix) except ClientError as ce: if ce.status in (404, ): self._container_exists() raiseCLIError(ce, details=[ 'To download an object, it must exist either as a file or' ' as a directory.', 'For example, to download everything under prefix/ the ' 'directory "prefix" must exist.', 'To see if an remote object is actually there:', ' kamaki file info [[pithos://UUID]/CONTAINER/]OBJECT', 'To create a directory object:', ' kamaki file mkdir [[pithos://UUID]/CONTAINER/]OBJECT']) if ce.status in (204, ): raise CLIError( 'No file or directory objects to download', details=[ 'To download a container (e.g., %s):' % self.container, ' kamaki container download %s [LOCAL_PATH]' % ( self.container)]) raise # We requested to download either a whole container or a directory if (not obj) or self.object_is_dir(obj): if self['recursive']: obj = obj or dict( name='', content_type='application/directory') dirs, files = [], [] result = self.client.container_get( prefix=prefix, if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date']) # Find the final local path for each remote object # [(remote name, final local path),.] for o in result.json: remote = o['name'] # First find the relative path of the object # without the prefix and any leading '/' relative = remote[len(prefix):].lstrip('/') # Translate it to a valid path with proper separator norm = relative.replace('/', path.sep) # Append it to the desired local path final = path.join(local_path, norm) if self.object_is_dir(o): dirs.append((remote, final)) else: files.append((remote, final)) self.error(r"%s -> %s" % (remote, final)) # Put the directories on top of the list for dpath in sorted(p for _, p in dirs): if path.exists(dpath): if path.isdir(dpath): continue raise CLIError( 'Cannot replace local file %s with a directory ' 'of the same name' % dpath, details=[ 'Either remove the file or specify a' 'different target location']) ret.append((None, dpath, None)) # Append the file objects for opath, lpath in files: if self['resume']: fxists = path.exists(lpath) if fxists and path.isdir(lpath): raise CLIError( 'Cannot change local dir %s into a file' % ( lpath), details=[ 'Either remove the file or specify a' 'different target location']) ret.append((opath, lpath, fxists)) elif path.exists(lpath): raise CLIError( 'Cannot overwrite %s' % lpath, details=['To overwrite/resume, use %s' % ( self.arguments['resume'].lvalue)]) else: ret.append((opath, lpath, None)) elif prefix: raise CLIError( 'Remote object /%s/%s is a directory' % ( self.container, prefix), details=['Use %s to download directories' % ( self.arguments['recursive'].lvalue)]) else: parsed_name = self.arguments['recursive'].lvalue raise CLIError( 'Cannot download container %s' % self.container, details=[ 'Use %s to download containers' % parsed_name, ' kamaki file download %s /%s [LOCAL_PATH]' % ( parsed_name, self.container)]) else: # Remote object is just a file # The local path to be stored already exists if path.exists(local_path): if not self['resume']: raise CLIError( 'Cannot overwrite local file %s' % (local_path), details=['To overwrite/resume, use %s' % ( self.arguments['resume'].lvalue)]) # The local path does not exist. elif path.sep in local_path: # Delegate intermediate local dir cration # to makedirs() inside _run() d = path.dirname(local_path) ret.append((None, d, None)) ret.append((prefix, local_path, self['resume'])) for r, l, resume in ret: if r: mode = 'rb+' if resume and path.exists(l) else 'wb+' with open(l, mode) as f: yield (r, f) else: yield (r, l) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container @errors.Pithos.object_path @errors.Pithos.local_path @errors.Pithos.local_path_download def _run(self, local_path): self.client.MAX_THREADS = int(self['max_threads'] or 5) progress_bar = None try: # From _src_dst(): # If rpath is None output_file is a directory. # If rpath is not None output_file is a file descriptor. for rpath, output_file in self._src_dst(local_path): # Create a directory if not rpath: if not path.exists(output_file): self.error('Create local directory %s' % output_file) makedirs(output_file) continue # Download a file self.error('/%s/%s --> %s' % ( self.container, rpath, output_file.name)) progress_bar, download_cb = self._safe_progress_bar( ' download') self.client.download_object( rpath, output_file, download_cb=download_cb, range_str=self['range'], version=self['object_version'], if_match=self['matching_etag'], resume=self['resume'], if_none_match=self['non_matching_etag'], if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date']) except KeyboardInterrupt: timeout = 0.5 msg = '\n' while activeCount() > 1: msg += 'Wait for %s threads: ' % (activeCount() - 1) self._err.write(msg) for thread in activethreads(): try: thread.join(timeout) self._err.write('.' if thread.isAlive() else '*') self._err.flush() except RuntimeError: continue finally: msg = '\b' * len(msg) timeout += 0.1 raise CLIError('Download canceled by user') finally: self._safe_progress_bar_finish(progress_bar) self.error('Download completed')
[docs] def main(self, remote_path_or_url, local_path=None): """ Dowload remote_path_or_url to local_path. """ super(self.__class__, self)._run(remote_path_or_url) # Translate relative remote path to local path with proper separator # and without trailing '/'. If not given use the name of the container rpath = self.path.rstrip('/').replace('/', path.sep) or self.container # If remote path is /pithos/dir1/dir2/ then here we download dir2 base = path.basename(rpath) # If local_path is not given use current dir if not local_path: local_path = path.join('.', base) # existing_dir/ -> existing_dir/base elif path.exists(local_path) and path.isdir(local_path): local_path = path.join(local_path, base) self._run(local_path=local_path)
@command(container_cmds)
[docs]class container_info(_PithosAccount, OptionalOutput): """Get information about a container""" arguments = dict( until_date=DateArgument('show metadata until then', '--until'), metadata=FlagArgument('Show only container metadata', '--metadata'), sizelimit=FlagArgument( 'Show the maximum size limit for container', '--size-limit'), in_bytes=FlagArgument('Show size limit in bytes', ('-b', '--bytes')) ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): if self['metadata']: r, preflen = dict(), len('x-container-meta-') for k, v in self.client.get_container_meta( until=self['until_date']).items(): r[k[preflen:]] = v elif self['sizelimit']: r = self.client.get_container_limit()['x-container-policy-quota'] r = {'size limit': 'unlimited' if r in ('0', ) else ( int(r) if self['in_bytes'] else format_size(r))} else: r = self.client.get_container_info() self.print_(r, self.print_dict)
[docs] def main(self, container): super(self.__class__, self)._run() self.container, self.client.container = container, container self._run()
[docs]class VersioningArgument(ValueArgument): schemes = ('auto', 'none') @property def value(self): return getattr(self, '_value', None) @value.setter
[docs] def value(self, new_scheme): if new_scheme: new_scheme = new_scheme.lower() if new_scheme not in self.schemes: raise CLIInvalidArgument('Invalid versioning value', details=[ 'Valid versioning values are %s' % ', '.join( self.schemes)]) self._value = new_scheme
@command(container_cmds)
[docs]class container_modify(_PithosAccount, OptionalOutput): """Modify the properties of a container""" arguments = dict( metadata_to_add=KeyValueArgument( 'Add metadata in the form KEY=VALUE (can be repeated)', '--metadata-add'), metadata_to_delete=RepeatableArgument( 'Delete metadata by KEY (can be repeated)', '--metadata-del'), sizelimit=DataSizeArgument( 'Set max size limit (0 for unlimited, ' 'use units B, KiB, KB, etc.)', '--size-limit'), versioning=VersioningArgument( 'Set a versioning scheme (%s)' % ', '.join( VersioningArgument.schemes), '--versioning') ) required = [ 'metadata_to_add', 'metadata_to_delete', 'sizelimit', 'versioning'] @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): metadata = self['metadata_to_add'] for k in (self['metadata_to_delete'] or []): metadata[k] = '' if metadata: self.client.set_container_meta(metadata) self.print_(self.client.get_container_meta(), self.print_dict) if self['sizelimit'] is not None: self.client.set_container_limit(self['sizelimit']) r = self.client.get_container_limit()['x-container-policy-quota'] r = 'unlimited' if r in ('0', ) else format_size(r) self.writeln('new size limit: %s' % r) if self['versioning']: self.client.set_container_versioning(self['versioning']) self.writeln('new versioning scheme: %s' % ( self.client.get_container_versioning()[ 'x-container-policy-versioning']))
[docs] def main(self, container): super(self.__class__, self)._run() self.client.container, self.container = container, container self._run()
@command(container_cmds)
[docs]class container_list(_PithosAccount, OptionalOutput, NameFilter): """List all containers, or their contents""" arguments = dict( detail=FlagArgument('Containers with details', ('-l', '--list')), limit=IntArgument('limit number of listed items', ('-n', '--number')), marker=ValueArgument('output greater that marker', '--marker'), modified_since_date=ValueArgument( 'show output modified since then', '--if-modified-since'), unmodified_since_date=ValueArgument( 'show output not modified since then', '--if-unmodified-since'), until_date=DateArgument('show metadata until then', '--until'), shared=FlagArgument('show only shared', '--shared'), more=FlagArgument('read long results', '--more'), enum=FlagArgument('Enumerate results', '--enumerate'), recursive=FlagArgument( 'Recursively list containers and their contents', ('-r', '--recursive')), shared_by_me=FlagArgument( 'show only files shared to other users', '--shared-by-me'), public=FlagArgument('show only published objects', '--public'), )
[docs] def print_containers(self, container_list): for index, container in enumerate(container_list): if 'bytes' in container: size = format_size(container['bytes']) prfx = ('%s. ' % (index + 1)) if self['enum'] else '' _cname = escape_ctrl_chars(container['name']) _cname = _cname if self['more'] else bold(_cname) cname = u'%s%s' % (prfx, _cname) if self['detail']: self.writeln(cname) pretty_c = container.copy() if 'bytes' in container: pretty_c['bytes'] = '%s (%s)' % (container['bytes'], size) self.print_dict(pretty_c, exclude=('name')) self.writeln() else: if 'count' in container and 'bytes' in container: self.writeln('%s (%s, %s objects)' % ( cname, size, container['count'])) else: self.writeln(cname) objects = container.get('objects', []) if objects: self.print_objects(objects) self.writeln('')
def _create_object_forest(self, container_list): try: for container in container_list: self.client.container = container['name'] objects = self.client.container_get( limit=False if self['more'] else self['limit'], if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date'], until=self['until_date'], show_only_shared=self['shared_by_me'], public=self['public']) container['objects'] = objects.json finally: self.client.container = None @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): container = self.container if container: r = self.client.container_get( limit=False if self['more'] else self['limit'], marker=self['marker'], if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date'], until=self['until_date'], show_only_shared=self['shared_by_me'], public=self['public']) else: r = self.client.account_get( limit=False if self['more'] else self['limit'], marker=self['marker'], if_modified_since=self['modified_since_date'], if_unmodified_since=self['unmodified_since_date'], until=self['until_date'], show_only_shared=self['shared_by_me'], public=self['public']) items = list(r.json or []) files = self._filter_by_name(items) if self['recursive'] and not container: self._create_object_forest(files) if self['more']: outbu, self._out = self._out, StringIO() try: if self['output_format']: self.print_(files) else: (self.print_objects if container else self.print_containers)( files) finally: if self['more']: pager(self._out.getvalue()) self._out = outbu
[docs] def main(self, container=None): super(self.__class__, self)._run() self.client.container, self.container = container, container self._run()
@command(container_cmds)
[docs]class container_create(_PithosAccount): """Create a new container""" arguments = dict( versioning=ValueArgument( 'set container versioning (auto/none)', '--versioning'), limit=IntArgument('set default container limit', '--limit'), meta=KeyValueArgument( 'set container metadata (can be repeated)', '--meta'), project_id=ValueArgument('assign container to project', '--project-id') ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): try: self.client.create_container( container=self.container, sizelimit=self['limit'], versioning=self['versioning'], project_id=self['project_id'], metadata=self['meta'], success=(201, )) except ClientError as ce: if ce.status in (202, ): raise CLIError( 'Container %s alread exists' % self.container, details=[ 'Delete %s or choose another name' % self.container]) elif self['project_id'] and ce.status in (400, 403, 404): self._project_id_exists(project_id=self['project_id']) raise
[docs] def main(self, new_container): super(self.__class__, self)._run() self.container, self.client.container = new_container, new_container self._run()
@command(container_cmds)
[docs]class container_delete(_PithosAccount): """Delete a container""" arguments = dict( yes=FlagArgument('Do not prompt for permission', '--yes'), recursive=FlagArgument( 'delete container even if not empty', ('-r', '--recursive')) ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): num_of_contents = int(self.client.get_container_info(self.container)[ 'x-container-object-count']) delimiter, msg = None, 'Delete container %s ?' % self.container if self['recursive']: delimiter, msg = '/', 'Empty and d%s' % msg[1:] elif num_of_contents: raise CLIError( 'Container %s is not empty' % self.container, details=[ 'Use %s to delete non-empty containers' % ( self.arguments['recursive'].lvalue)]) if self['yes'] or self.ask_user(msg): if num_of_contents: self.client.del_container(delimiter=delimiter) self.client.purge_container()
[docs] def main(self, container): super(self.__class__, self)._run() self.container, self.client.container = container, container self._run()
@command(container_cmds)
[docs]class container_empty(_PithosAccount): """Empty a container""" arguments = dict(yes=FlagArgument('Do not prompt for permission', '--yes')) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): if self['yes'] or self.ask_user( 'Empty container %s ?' % self.container): self.client.del_container(delimiter='/')
[docs] def main(self, container): super(self.__class__, self)._run() self.container, self.client.container = container, container self._run()
@command(container_cmds)
[docs]class container_reassign(_PithosAccount): """Assign a container to a different project""" arguments = dict( project_id=ValueArgument('The project to assign', '--project-id') ) required = ('project_id', ) @errors.Generic.all @errors.Pithos.connection @errors.Pithos.container def _run(self): try: self.client.reassign_container(self['project_id']) except ClientError as ce: if ce.status in (400, 403, 404): self._project_id_exists(project_id=self['project_id']) raise
[docs] def main(self, container): super(self.__class__, self)._run() self.container, self.client.container = container, container self._run()
@command(sharer_cmds)
[docs]class sharer_list(_PithosAccount, OptionalOutput): """List accounts who share file objects with current user""" arguments = dict( detail=FlagArgument('show detailed output', ('-l', '--details')), marker=ValueArgument('show output greater than marker', '--marker') ) @errors.Generic.all @errors.Pithos.connection def _run(self): accounts = self.client.get_sharing_accounts(marker=self['marker']) if not self['output_format']: usernames = self._uuids2usernames( [acc['name'] for acc in accounts]) for item in accounts: uuid = item['name'] item['id'], item['name'] = uuid, usernames[uuid] if not self['detail']: item.pop('last_modified') self.print_(accounts)
[docs] def main(self): super(self.__class__, self)._run() self._run()
@command(sharer_cmds)
[docs]class sharer_info(_PithosAccount, OptionalOutput): """Details on a Pithos+ sharer account (default: current account)""" @errors.Generic.all @errors.Pithos.connection def _run(self): self.print_(self.client.get_account_info(), self.print_dict)
[docs] def main(self, account_uuid_or_name=None): super(self.__class__, self)._run() if account_uuid_or_name: arg = UserAccountArgument('Check', ' ') arg.account_client = self.astakos arg.value = account_uuid_or_name self.client.account, self.account = arg.value, arg.value self._run()
class _PithosGroup(_PithosAccount): prefix = 'x-account-group-' preflen = len(prefix) def _groups(self): groups = dict() for k, v in self.client.get_account_group().items(): groups[k[self.preflen:]] = v return groups @command(group_cmds)
[docs]class group_list(_PithosGroup, OptionalOutput): """list all groups and group members""" @errors.Generic.all @errors.Pithos.connection def _run(self): self.print_(self._groups(), self.print_dict)
[docs] def main(self): super(self.__class__, self)._run() self._run()
@command(group_cmds)
[docs]class group_create(_PithosGroup, OptionalOutput): """Create a group of users""" arguments = dict( user_uuid=RepeatableArgument('Add a user to the group', '--uuid'), username=RepeatableArgument('Add a user to the group', '--username') ) required = ['user_uuid', 'username'] @errors.Generic.all @errors.Pithos.connection def _run(self, groupname, *users): if groupname in self._groups() and not self.ask_user( 'Group %s already exists, overwrite?' % groupname): self.error('Aborted') else: self.client.set_account_group(groupname, users) self.print_(self._groups(), self.print_dict)
[docs] def main(self, groupname): super(self.__class__, self)._run() users = (self['user_uuid'] or []) + self._usernames2uuids( self['username'] or []).values() if users: self._run(groupname, *users) else: raise CLISyntaxError( 'No valid users specified, use %s or %s' % ( self.arguments['user_uuid'].lvalue, self.arguments['username'].lvalue), details=[ 'Check if a username or uuid is valid with', ' user uuid2username', 'OR', ' user username2uuid'])
@command(group_cmds)
[docs]class group_delete(_PithosGroup, OptionalOutput): """Delete a user group""" @errors.Generic.all @errors.Pithos.connection def _run(self, groupname): self.client.del_account_group(groupname) self.print_(self._groups(), self.print_dict)
[docs] def main(self, groupname): super(self.__class__, self)._run() self._run(groupname)