Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-06-25 08:26
📄 __init__.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Package marker file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals	797 B	2025-06-25 08:19
📄 acl.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of acl command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from apitools.base.py import encoding from gslib import metrics from gslib import gcs_json_api from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.storage_url import RaiseErrorIfUrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil acl set [-f] [-r] [-a] (<file-path>\|<predefined-acl>) url... """ _GET_SYNOPSIS = """ gsutil acl get url """ _CH_SYNOPSIS = """ gsutil acl ch [-f] [-r] <grant>... url... where each <grant> is one of the following forms: -u <id>\|<email>:<permission> -g <id>\|<email>\|<domain>\|All\|AllAuth:<permission> -p (viewers\|editors\|owners)-<project number>:<permission> -d <id>\|<email>\|<domain>\|All\|AllAuth\|(viewers\|editors\|owners)-<project number> """ _GET_DESCRIPTION = """ <B>GET</B> The ``acl get`` command gets the ACL text for a bucket or object, which you can save and edit for the acl set command. """ _SET_DESCRIPTION = """ <B>SET</B> The ``acl set`` command allows you to set an Access Control List on one or more buckets and objects. As part of the command, you must specify either a predefined ACL or the path to a file that contains ACL text. The simplest way to use the ``acl set`` command is to specify one of the predefined ACLs, e.g.,: gsutil acl set private gs://example-bucket/example-object If you want to make an object or bucket publicly readable or writable, it is recommended to use ``acl ch``, to avoid accidentally removing OWNER permissions. See the ``acl ch`` section for details. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. If you want to define more fine-grained control over your data, you can retrieve an ACL using the ``acl get`` command, save the output to a file, edit the file, and then use the ``acl set`` command to set that ACL on the buckets and/or objects. For example: gsutil acl get gs://bucket/file.txt > acl.txt Make changes to acl.txt such as adding an additional grant, then: gsutil acl set acl.txt gs://cats/file.txt Note that you can set an ACL on multiple buckets or objects at once. For example, to set ACLs on all .jpg files found in a bucket: gsutil acl set acl.txt gs://bucket/.jpg If you have a large number of ACLs to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m acl set acl.txt gs://bucket/.jpg Note that multi-threading/multi-processing is only done when the named URLs refer to objects, which happens either if you name specific objects or if you enumerate objects by using an object wildcard or specifying the acl -r flag. <B>SET OPTIONS</B> The "set" sub-command has the following options -R, -r Performs "acl set" request recursively, to all objects under the specified URL. -a Performs "acl set" request on all object versions. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. If some of the ACLs couldn't be set, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m acl...". """ _CH_DESCRIPTION = """ <B>CH</B> The "acl ch" (or "acl change") command updates access control lists, similar in spirit to the Linux chmod command. You can specify multiple access grant additions and deletions in a single command run; all changes will be made atomically to each object in turn. For example, if the command requests deleting one grant and adding a different grant, the ACLs being updated will never be left in an intermediate state where one grant has been deleted but the second grant not yet added. Each change specifies a user or group grant to add or delete, and for grant additions, one of R, W, O (for the permission to be granted). A more formal description is provided in a later section; below we provide examples. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Grant anyone on the internet READ access to the object example-object: gsutil acl ch -u allUsers:R gs://example-bucket/example-object NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Grant the user john.doe@example.com READ access to all objects in example-bucket that begin with folder/: gsutil acl ch -r -u john.doe@example.com:R gs://example-bucket/folder/ Grant the group admins@example.com OWNER access to all jpg files in example-bucket: gsutil acl ch -g admins@example.com:O gs://example-bucket/**.jpg Grant the owners of project example-project WRITE access to the bucket example-bucket: gsutil acl ch -p owners-example-project:W gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. Remove access to the bucket example-bucket for the viewers of project number 12345: gsutil acl ch -d viewers-12345 gs://example-bucket NOTE: You cannot remove the project owners group from ACLs of gs:// buckets in the given project. Attempts to do so will appear to succeed, but the service will add the project owners group into the new set of ACLs before applying it. Note that removing a project requires you to reference the project by its number (which you can see with the acl get command) as opposed to its project ID string. Grant the service account foo@developer.gserviceaccount.com WRITE access to the bucket example-bucket: gsutil acl ch -u foo@developer.gserviceaccount.com:W gs://example-bucket Grant all users from the `G Suite <https://www.google.com/work/apps/business/>`_ domain my-domain.org READ access to the bucket gcs.my-domain.org: gsutil acl ch -g my-domain.org:R gs://gcs.my-domain.org Remove any current access by john.doe@example.com from the bucket example-bucket: gsutil acl ch -d john.doe@example.com gs://example-bucket If you have a large number of objects to update, enabling multi-threading with the gsutil -m flag can significantly improve performance. The following command adds OWNER for admin@example.org using multi-threading: gsutil -m acl ch -r -u admin@example.org:O gs://example-bucket Grant READ access to everyone from my-domain.org and to all authenticated users, and grant OWNER to admin@mydomain.org, for the buckets my-bucket and my-other-bucket, with multi-threading enabled: gsutil -m acl ch -r -g my-domain.org:R -g AllAuth:R \\ -u admin@mydomain.org:O gs://my-bucket/ gs://my-other-bucket <B>CH ROLES</B> You may specify the following roles with either their shorthand or their full name: R: READ W: WRITE O: OWNER For more information on these roles and the access they grant, see the permissions section of the `Access Control Lists page <https://cloud.google.com/storage/docs/access-control/lists#permissions>`_. <B>CH ENTITIES</B> There are four different entity types: Users, Groups, All Authenticated Users, and All Users. Users are added with -u and a plain ID or email address, as in "-u john-doe@gmail.com:r". Note: Service Accounts are considered to be users. Groups are like users, but specified with the -g flag, as in "-g power-users@example.com:O". Groups may also be specified as a full domain, as in "-g my-company.com:r". allAuthenticatedUsers and allUsers are specified directly, as in "-g allUsers:R" or "-g allAuthenticatedUsers:O". These are case insensitive, and may be shortened to "all" and "allauth", respectively. Removing roles is specified with the -d flag and an ID, email address, domain, or one of allUsers or allAuthenticatedUsers. Many entities' roles can be specified on the same command line, allowing bundled changes to be executed in a single run. This will reduce the number of requests made to the server. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -R, -r Performs acl ch request recursively, to all objects under the specified URL. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = (""" The acl command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION])) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) def _ApplyExceptionHandler(cls, exception): cls.logger.error('Encountered a problem: %s', exception) cls.everything_set_okay = False def _ApplyAclChangesWrapper(cls, url_or_expansion_result, thread_state=None): cls.ApplyAclChanges(url_or_expansion_result, thread_state=thread_state) class AclCommand(Command): """Implementation of gsutil acl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'acl', command_name_aliases=['getacl', 'setacl', 'chacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument() ], 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='acl', help_name_aliases=['getacl', 'setacl', 'chmod', 'chacl'], help_type='command_help', help_one_line_summary='Get, set, or change bucket and/or object ACLs', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text }, ) def _get_shim_command_group(self): object_or_bucket_urls = [StorageUrlFromString(url) for url in self.args] recurse = False for (flag_key, _) in self.sub_opts: if flag_key in ('-r', '-R'): recurse = True break RaiseErrorIfUrlsAreMixOfBucketsAndObjects(object_or_bucket_urls, recurse) if object_or_bucket_urls[0].IsBucket() and not recurse: return 'buckets' else: return 'objects' def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'describe', '--format=multi(acl:format=json)' ], flag_map={}) elif sub_command == 'set': # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() acl_file_or_predefined_acl = self.args.pop(0) if os.path.isfile(acl_file_or_predefined_acl): acl_flag = '--acl-file=' + acl_file_or_predefined_acl else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl acl_flag = '--predefined-acl=' + predefined_acl command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'] + [acl_flag], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) command_group = self._get_shim_command_group() gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', command_group, 'update'], flag_map={ '-g': GcloudStorageFlag('--add-acl-grant'), '-p': GcloudStorageFlag('--add-acl-grant'), '-u': GcloudStorageFlag('--add-acl-grant'), '-d': GcloudStorageFlag('--remove-acl-grant'), '-a': GcloudStorageFlag('--all-versions'), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set') or (self.command_alias_used == 'setacl'): return 1 else: return 0 def _SetAcl(self): """Parses options and sets ACLs on the specified buckets/objects.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException as unused_e: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _ChAcl(self): """Parses options and changes ACLs on the specified buckets/objects.""" self.parse_versions = True self.changes = [] self.continue_on_error = False if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-g': if 'gserviceaccount.com' in a: raise CommandException( 'Service accounts are considered users, not groups; please use ' '"gsutil acl ch -u" instead of "gsutil acl ch -g"') self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) elif o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) elif o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) elif o == '-d': self.changes.append(acl_helper.AclDel(a)) elif o == '-r' or o == '-R': self.recursion_requested = True else: self.RaiseInvalidArgumentException() if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) self.everything_set_okay = True self.ApplyAclFunc(_ApplyAclChangesWrapper, _ApplyExceptionHandler, self.args, object_fields=['acl', 'generation', 'metageneration']) if not self.everything_set_okay: raise CommandException('ACLs for some objects could not be set.') def _RaiseForAccessDenied(self, url): self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, name_expansion_result, thread_state=None): """Applies the changes in self.changes to the provided URL. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, gsutil Cloud API instance to apply the changes. """ if thread_state: gsutil_api = thread_state else: gsutil_api = self.gsutil_api url = name_expansion_result.expanded_storage_url if url.IsBucket(): bucket = gsutil_api.GetBucket(url.bucket_name, provider=url.scheme, fields=['acl', 'metageneration']) current_acl = bucket.acl elif url.IsObject(): gcs_object = encoding.JsonToMessage(apitools_messages.Object, name_expansion_result.expanded_result) current_acl = gcs_object.acl if not current_acl: self._RaiseForAccessDenied(url) if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return try: if url.IsBucket(): preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(acl=current_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) else: # Object preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) object_metadata = apitools_messages.Object(acl=current_acl) try: gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=url.generation, fields=['id']) except PreconditionException as e: # Special retry case where we want to do an additional step, the read # of the read-modify-write cycle, to fetch the correct object # metadata before reattempting ACL changes. self._RefetchObjectMetadataAndApplyAclChanges(url, gsutil_api) self.logger.info('Updated ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._RaiseForAccessDenied(url) except PreconditionException as e: # For objects, retry attempts should have already been handled. if url.IsObject(): raise CommandException(str(e)) # For buckets, raise PreconditionException and continue to next retry. raise e @Retry(PreconditionException, tries=3, timeout_secs=1) def _RefetchObjectMetadataAndApplyAclChanges(self, url, gsutil_api): """Reattempts object ACL changes after a PreconditionException.""" gcs_object = gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=url.scheme, fields=['acl', 'generation', 'metageneration']) current_acl = gcs_object.acl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return object_metadata = apitools_messages.Object(acl=current_acl) preconditions = Preconditions(gen_match=gcs_object.generation, meta_gen_match=gcs_object.metageneration) gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, preconditions=preconditions, provider=url.scheme, generation=gcs_object.generation, fields=['id']) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, acl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, acl_message, 'acl', self.logger) return modification_count def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self.GetAndPrintAcl(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetAcl() elif action_subcommand in ('ch', 'change'): metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChAcl() else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help acl".') % (action_subcommand, self.command_name)) return 0	23.78 KB	2025-06-25 08:19
📄 autoclass.py # -- coding: utf-8 -- # Copyright 2021 Google LLC All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the autoclass command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil autoclass set (on\|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil autoclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command requires an additional sub-command, either ``on`` or ``off``, which enables or disables Autoclass for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The ``get`` sub-command gets the current Autoclass configuration for a bucket. The returned configuration has the following fields: ``enabled``: a boolean field indicating whether the feature is on or off. ``toggleTime``: a timestamp indicating when the enabled field was set. """ _DESCRIPTION = """ The `Autoclass <https://cloud.google.com/storage/docs/autoclass>`_ feature automatically selects the best storage class for objects based on access patterns. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("gs://{}:", name),' + ' format(" Enabled: {}",' + 'autoclass.enabled.yesno(True, False)),' + ' format(" Toggle Time: {}",' + 'autoclass.toggleTime))') class AutoclassCommand(Command): """Implements the gsutil autoclass command.""" command_spec = Command.CreateCommandSpec( 'autoclass', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='autoclass', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Autoclass feature', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--enable-autoclass', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-enable-autoclass', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _get_autoclass(self, blr): """Gets the autoclass setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['autoclass'], provider=bucket_url.scheme) bucket = str(bucket_url).rstrip('/') if bucket_metadata.autoclass: enabled = getattr(bucket_metadata.autoclass, 'enabled', False) toggle_time = getattr(bucket_metadata.autoclass, 'toggleTime', None) else: enabled = False toggle_time = None print('{}:\n' ' Enabled: {}\n' ' Toggle Time: {}'.format(bucket, enabled, toggle_time)) def _set_autoclass(self, blr, setting_arg): """Turns autoclass on or off for a bucket.""" bucket_url = blr.storage_url autoclass_config = apitools_messages.Bucket.AutoclassValue() autoclass_config.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(autoclass=autoclass_config) print('Setting Autoclass %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['autoclass'], provider=bucket_url.scheme) return 0 def _autoclass(self): """Handles autoclass command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('autoclass only supports get\|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._get_autoclass elif subcommand == 'set': subcommand_func = self._set_autoclass setting_arg = self.args.pop(0) text_util.InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the autoclass command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._autoclass() else: raise CommandException('Invalid subcommand "%s", use get\|set instead.' % action_subcommand)	8.89 KB	2025-06-25 08:19
📄 bucketpolicyonly.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands import ubla from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.text_util import InsistOnOrOff _SET_SYNOPSIS = """ gsutil bucketpolicyonly set (on\|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil bucketpolicyonly get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``bucketpolicyonly set`` command enables or disables the uniform bucket-level access feature on Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil bucketpolicyonly set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil bucketpolicyonly set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``bucketpolicyonly get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket. <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil bucketpolicyonly get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The Bucket Policy Only feature is now known as `uniform bucket-level access <https://cloud.google.com/storage/docs/uniform-bucket-level-access>`_. The ``bucketpolicyonly`` command is still supported, but we recommend using the equivalent ``ubla`` `command <https://cloud.google.com/storage/docs/gsutil/commands/ubla>`_. The ``bucketpolicyonly`` command is used to retrieve or configure the uniform bucket-level access setting of Cloud Storage buckets. This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class BucketPolicyOnlyCommand(Command): """Implements the gsutil bucketpolicyonly command.""" command_spec = Command.CreateCommandSpec( 'bucketpolicyonly', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='bucketpolicyonly', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) # The get subcommand works exactly like ubla except we want to replace # "Uniform bucket-level access" with the bpo text. gcloud_storage_map = ubla.UblaCommand.gcloud_storage_map format_flag = gcloud_storage_map.gcloud_command['get'].gcloud_command[3] gcloud_storage_map.gcloud_command['get'].gcloud_command[ 3] = format_flag.replace('Uniform bucket-level access', 'Bucket Policy Only') def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetBucketPolicyOnly(self, blr): """Gets the Bucket Policy Only setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration bucket_policy_only = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': bucket_policy_only.enabled } locked_time_line = '' if bucket_policy_only.lockedTime: fields['locked_time'] = bucket_policy_only.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if bucket_policy_only: print(('Bucket Policy Only setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(*fields)) def _SetBucketPolicyOnly(self, blr, setting_arg): """Sets the Bucket Policy Only setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Bucket Policy Only for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _BucketPolicyOnly(self): """Handles bucketpolicyonly command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('bucketpolicyonly only supports get\|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetBucketPolicyOnly elif subcommand == 'set': subcommand_func = self._SetBucketPolicyOnly setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the bucketpolicyonly command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._BucketPolicyOnly() else: raise CommandException('Invalid subcommand "%s", use get\|set instead.' % action_subcommand)	8.7 KB	2025-06-25 08:19
📄 cat.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import cat_helper from gslib.utils import constants from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap if six.PY3: long = int _SYNOPSIS = """ gsutil cat [-h] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The cat command outputs the contents of one or more URLs to stdout. While the cat command does not compute a checksum, it is otherwise equivalent to doing: gsutil cp url... - (The final '-' causes gsutil to stream the output to stdout.) WARNING: The gsutil cat command does not compute a checksum of the downloaded data. Therefore, we recommend that users either perform their own validation of the output of gsutil cat or use gsutil cp or rsync (both of which perform integrity checking automatically). <B>OPTIONS</B> -h Prints short header for each object. For example: gsutil cat -h gs://bucket/meeting_notes/2012_Feb/.txt This would print a header with the object name before the contents of each text object that matched the wildcard. -r range Causes gsutil to output just the specified byte range of the object. Ranges can be of these forms: start-end (e.g., -r 256-5939) start- (e.g., -r 256-) -numbytes (e.g., -r -5) where offsets start at 0, start-end means to return bytes start through end (inclusive), start- means to return bytes start through the end of the object, and -numbytes means to return the last numbytes of the object. For example: gsutil cat -r 256-939 gs://bucket/object returns bytes 256 through 939, while: gsutil cat -r -5 gs://bucket/object returns the final 5 bytes of the object. """) class CatCommand(Command): """Implementation of gsutil cat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='hr:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Concatenate object content to stdout', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cat'], flag_map={ '-h': GcloudStorageFlag('-d'), '-r': GcloudStorageFlag('-r'), }, ) # Command entry point. def RunCommand(self): """Command entry point for the cat command.""" show_header = False request_range = None start_byte = 0 end_byte = None if self.sub_opts: for o, a in self.sub_opts: if o == '-h': show_header = True elif o == '-r': request_range = a.strip() # This if statement ensures the full object is returned # instead of throwing a CommandException. if request_range == '-': continue range_matcher = re.compile( '^(?P<start>[0-9]+)-(?P<end>[0-9])$\|^(?P<endslice>-[0-9]+)$') range_match = range_matcher.match(request_range) if not range_match: raise CommandException('Invalid range (%s)' % request_range) if range_match.group('start'): start_byte = long(range_match.group('start')) if range_match.group('end'): end_byte = long(range_match.group('end')) if range_match.group('endslice'): start_byte = long(range_match.group('endslice')) else: self.RaiseInvalidArgumentException() return cat_helper.CatHelper(self).CatUrlStrings(self.args, show_header=show_header, start_byte=start_byte, end_byte=end_byte)	5.43 KB	2025-06-25 08:19
📄 compose.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of compose command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto import config from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_COMPOSE_ARITY = 32 _SYNOPSIS = """ gsutil compose gs://bucket/source_obj1 [gs://bucket/source_obj2 ...] gs://bucket/composite_obj """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The compose command creates a new object whose content is the concatenation of a given sequence of source objects under the same bucket. gsutil uses the content type of the first source object to determine the destination object's content type and does not modify or delete the source objects as part of the compose command. For more information, see the `composite objects topic <https://cloud.google.com/storage/docs/composite-objects>`_. There is a limit (currently %d) to the number of components that can be composed in a single operation. """ % (MAX_COMPOSE_ARITY)) class ComposeCommand(Command): """Implementation of gsutil compose command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'compose', command_name_aliases=['concat'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=MAX_COMPOSE_ARITY + 1, supported_sub_args='', # Not files, just object names without gs:// prefix. file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='compose', help_name_aliases=['concat'], help_type='command_help', help_one_line_summary=( 'Concatenate a sequence of objects into a new composite object.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'compose'], flag_map={}, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"compose" called on URL with unsupported provider (%s).' % str(url)) # Command entry point. def RunCommand(self): """Command entry point for the compose command.""" target_url_str = self.args[-1] self.args = self.args[:-1] target_url = StorageUrlFromString(target_url_str) self.CheckProvider(target_url) if target_url.HasGeneration(): raise CommandException('A version-specific URL (%s) cannot be ' 'the destination for gsutil compose - abort.' % target_url) dst_obj_metadata = apitools_messages.Object(name=target_url.object_name, bucket=target_url.bucket_name) components = [] # Remember the first source object so we can get its content type. first_src_url = None for src_url_str in self.args: if ContainsWildcard(src_url_str): src_url_iter = self.WildcardIterator(src_url_str).IterObjects() else: src_url_iter = [BucketListingObject(StorageUrlFromString(src_url_str))] for blr in src_url_iter: src_url = blr.storage_url self.CheckProvider(src_url) if src_url.bucket_name != target_url.bucket_name: raise CommandException('GCS does not support inter-bucket composing.') if not first_src_url: first_src_url = src_url src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=src_url.object_name)) if src_url.HasGeneration(): src_obj_metadata.generation = int(src_url.generation) components.append(src_obj_metadata) # Avoid expanding too many components, and sanity check each name # expansion result. if len(components) > MAX_COMPOSE_ARITY: raise CommandException('"compose" called with too many component ' 'objects. Limit is %d.' % MAX_COMPOSE_ARITY) if not components: raise CommandException('"compose" requires at least 1 component object.') first_src_obj_metadata = self.gsutil_api.GetObjectMetadata( first_src_url.bucket_name, first_src_url.object_name, provider=first_src_url.scheme, fields=['contentEncoding', 'contentType']) dst_obj_metadata.contentType = first_src_obj_metadata.contentType dst_obj_metadata.contentEncoding = first_src_obj_metadata.contentEncoding preconditions = PreconditionsFromHeaders(self.headers or {}) self.logger.info('Composing %s from %d component object(s).', target_url, len(components)) self.gsutil_api.ComposeObject( components, dst_obj_metadata, preconditions=preconditions, provider=target_url.scheme, encryption_tuple=GetEncryptionKeyWrapper(config))	6.28 KB	2025-06-25 08:19
📄 config.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of config command for creating a gsutil configuration file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import json import multiprocessing import os import signal import socket import stat import sys import textwrap import time import webbrowser from six.moves import input from six.moves.http_client import ResponseNotReady import boto from boto.provider import Provider import gslib from gslib.command import Command from gslib.command import DEFAULT_TASK_ESTIMATION_THRESHOLD from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.cred_types import CredTypes from gslib.exception import AbortException from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import constants from gslib.utils import system_util from gslib.utils.hashing_helper import CHECK_HASH_ALWAYS from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_SKIP from gslib.utils.hashing_helper import CHECK_HASH_NEVER from gslib.utils.parallelism_framework_util import ShouldProhibitMultiprocessing from httplib2 import ServerNotFoundError from oauth2client.client import HAS_CRYPTO _SYNOPSIS = """ gsutil [-D] config (-a\|-e) [-n] [-o <file>] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil config`` command generally applies to users who have legacy standalone installations of gsutil. If you installed gsutil via the Cloud SDK, ``gsutil config`` fails unless you are specifically using the ``-a`` flag or have configured gcloud to not pass its managed credentials to gsutil (via the command ``gcloud config set pass_credentials_to_gsutil false``). For all other use cases, Cloud SDK users should use the ``gcloud auth`` group of commands instead, which configures OAuth2 credentials that gcloud implicitly passes to gsutil at runtime. To check if you are using gsutil from the Cloud SDK or as a legacy standalone, use ``gsutil version -l`` and in the output look for "using cloud sdk". Important: The default behavior for the ``gsutil config`` command is to obtain user account credentials for authentication. However, user account credentials are no longer supported for standalone gsutil. For this reason, running the default ``gsutil config`` command fails, and using any of the following flags causes the command to fail: ``-b``, ``-f``, ``-r``, ``--reauth``, ``-s``, ``-w``. When using standalone gsutil, it's recommended that you use service account credentials via the ``-e`` flag. The ``gsutil config`` command obtains access credentials for Cloud Storage and writes a `boto/gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_ containing the obtained credentials along with a number of other configuration- controllable values. Unless specified otherwise (see OPTIONS), the configuration file is written to ~/.boto (i.e., the file .boto under the user's home directory). If the default file already exists, an attempt is made to rename the existing file to ~/.boto.bak; if that attempt fails the command exits. A different destination file can be specified with the ``-o`` option (see OPTIONS). Because the boto configuration file contains your credentials you should keep its file permissions set so no one but you has read access. (The file is created read-only when you run ``gsutil config``.) <B>CONFIGURING SERVICE ACCOUNT CREDENTIALS</B> Service accounts are useful for authenticating on behalf of a service or application (as opposed to a user). If you use gsutil as a legacy stand-alone tool, you configure credentials for service accounts using the ``-e`` option: gsutil config -e Note that if you use gsutil through the Cloud SDK, you instead activate your service account via the `gcloud auth activate-service-account <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ command. When you run ``gsutil config -e``, you are prompted for the path to your private key file and, if not using a JSON key file, your service account email address and key file password. To get this data, follow the instructions on `Service Accounts <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_. Using this information, gsutil populates the "gs_service_key_file" attribute in the boto configuration file. If not using a JSON key file, gsutil also populates the "gs_service_client_id" and "gs_service_key_file_password" attributes. Note that your service account is NOT considered an Owner for the purposes of API access (see "gsutil help creds" for more information about this). See https://developers.google.com/identity/protocols/OAuth2ServiceAccount for further information on service account authentication. If you want to use credentials based on access key and secret (the older authentication method before OAuth2 was supported), see the ``-a`` option in the OPTIONS section. If you wish to use gsutil with other providers (or to copy data back and forth between multiple providers) you can edit their credentials into the [Credentials] section after creating the initial boto configuration file. <B>OPTIONS</B> -a Prompt for Google Cloud Storage access key and secret (the older authentication method before OAuth2 was supported) instead of obtaining an OAuth2 token. -e Prompt for service account credentials. This option requires that ``-a`` is not set. -n Write the configuration file without authentication configured. This flag is mutually exclusive with all flags other than ``-o``. -o <file> Write the configuration to <file> instead of ~/.boto. Use ``-`` for stdout. """) GOOG_CLOUD_CONSOLE_URI = 'https://cloud.google.com/console#/project' CONFIG_PRELUDE_CONTENT = """ # This file contains credentials and other configuration information needed # by the boto library, used by gsutil. You can edit this file (e.g., to add # credentials) but be careful not to mis-edit any of the variable names (like # "gs_access_key_id") or remove important markers (like the "[Credentials]" and # "[Boto]" section delimiters). # """ # Default number of OS processes and Python threads for parallel operations. # On Linux systems we automatically scale the number of processes to match # the underlying CPU/core count. Given we'll be running multiple concurrent # processes on a typical multi-core Linux computer, to avoid being too # aggressive with resources, the default number of threads is reduced from # the previous value of 24 to 5. # # We also cap the maximum number of default processes at 32. Since each level # of recursion depth gets its own process pool, this means a maximum of # 64 processes with the current maximum recursion depth of 2. We limit this # number because testing with more than 200 processes showed fatal locking # exceptions in Python's multiprocessing.Manager. More processes are # probably not needed to saturate most networks. # # On Windows and Alpine Linux, Python multi-processing presents various # challenges so we retain compatibility with the established parallel mode # operation, i.e. one process and 24 threads. should_prohibit_multiprocessing, unused_os = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing: DEFAULT_PARALLEL_PROCESS_COUNT = 1 DEFAULT_PARALLEL_THREAD_COUNT = 24 else: DEFAULT_PARALLEL_PROCESS_COUNT = min(multiprocessing.cpu_count(), 32) DEFAULT_PARALLEL_THREAD_COUNT = 5 # TODO: Once compiled crcmod is being distributed by major Linux distributions # revert DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD value to '150M'. DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD = '0' DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE = '50M' DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD = '150M' DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE = '200M' DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS = 4 # Compressed transport encoded uploads buffer chunks of compressed data. When # running many uploads in parallel, compression may consume more memory than # available. This restricts the number of compressed transport encoded uploads # running in parallel such that they don't consume more memory than set here. DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = '2G' # gzip compression level. This is simply making the python stdlib default explicit. DEFAULT_GZIP_COMPRESSION_LEVEL = 9 CONFIG_BOTO_SECTION_CONTENT = """ [Boto] # http_socket_timeout specifies the timeout (in seconds) used to tell httplib # how long to wait for socket timeouts. The default is 70 seconds. Note that # this timeout only applies to httplib, not to httplib2 (which is used for # OAuth2 refresh/access token exchanges). #http_socket_timeout = 70 # The following two options control the use of a secure transport for requests # to S3 and Google Cloud Storage. It is highly recommended to set both options # to True in production environments, especially when using OAuth2 bearer token # authentication with Google Cloud Storage. # Set 'https_validate_certificates' to False to disable server certificate # checking. The default for this option in the boto library is currently # 'False' (to avoid breaking apps that depend on invalid certificates); it is # therefore strongly recommended to always set this option explicitly to True # in configuration files, to protect against "man-in-the-middle" attacks. https_validate_certificates = True # 'debug' controls the level of debug messages printed for the XML API only: # 0 for none, 1 for basic boto debug, 2 for all boto debug plus HTTP # requests/responses. #debug = <0, 1, or 2> # 'num_retries' controls the number of retry attempts made when errors occur # during data transfers. The default is 6. # Note 1: You can cause gsutil to retry failures effectively infinitely by # setting this value to a large number (like 10000). Doing that could be useful # in cases where your network connection occasionally fails and is down for an # extended period of time, because when it comes back up gsutil will continue # retrying. However, in general we recommend not setting the value above 10, # because otherwise gsutil could appear to "hang" due to excessive retries # (since unless you run gsutil -D you won't see any logged evidence that gsutil # is retrying). # Note 2: Don't set this value to 0, as it will cause boto to fail when reusing # HTTP connections. #num_retries = <integer value> # 'max_retry_delay' controls the max delay (in seconds) between retries. The # default value is 60, so the backoff sequence will be 1 seconds, 2 seconds, 4, # 8, 16, 32, and then 60 for all subsequent retries for a given HTTP request. # Note: At present this value only impacts the XML API and the JSON API uses a # fixed value of 60. #max_retry_delay = <integer value> """ CONFIG_GOOGLECOMPUTE_SECTION_CONTENT = """ [GoogleCompute] # 'service_account' specifies the a Google Compute Engine service account to # use for credentials. This value is intended for use only on Google Compute # Engine virtual machines and usually lives in /etc/boto.cfg. Most users # shouldn't need to edit this part of the config. #service_account = default """ CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT = """ [GSUtil] # 'resumable_threshold' specifies the smallest file size [bytes] for which # resumable Google Cloud Storage uploads are attempted. The default is 8388608 # (8 MiB). #resumable_threshold = %(resumable_threshold)d # 'rsync_buffer_lines' specifies the number of lines of bucket or directory # listings saved in each temp file during sorting. (The complete set is # split across temp files and separately sorted/merged, to avoid needing to # fit everything in memory at once.) If you are trying to synchronize very # large directories/buckets (e.g., containing millions or more objects), # having too small a value here can cause gsutil to run out of open file # handles. If that happens, you can try to increase the number of open file # handles your system allows (e.g., see 'man ulimit' on Linux; see also # http://docs.python.org/2/library/resource.html). If you can't do that (or # if you're already at the upper limit), increasing rsync_buffer_lines will # cause gsutil to use fewer file handles, but at the cost of more memory. With # rsync_buffer_lines set to 32000 and assuming a typical URL is 100 bytes # long, gsutil will require approximately 10 MiB of memory while building # the synchronization state, and will require approximately 60 open file # descriptors to build the synchronization state over all 1M source and 1M # destination URLs. Memory and file descriptors are only consumed while # building the state; once the state is built, it resides in two temp files that # are read and processed incrementally during the actual copy/delete # operations. #rsync_buffer_lines = 32000 # 'state_dir' specifies the base location where files that # need a static location are stored, such as pointers to credentials, # resumable transfer tracker files, and the last software update check. # By default these files are stored in ~/.gsutil #state_dir = <file_path> # gsutil periodically checks whether a new version of the gsutil software is # available. 'software_update_check_period' specifies the number of days # between such checks. The default is 30. Setting the value to 0 disables # periodic software update checks. #software_update_check_period = 30 # 'tab_completion_timeout' controls the timeout (in seconds) for tab # completions that involve remote requests (such as bucket or object names). # If tab completion does not succeed within this timeout, no tab completion # suggestions will be returned. # A value of 0 will disable completions that involve remote requests. #tab_completion_timeout = 5 # 'parallel_process_count' and 'parallel_thread_count' specify the number # of OS processes and Python threads, respectively, to use when executing # operations in parallel. The default settings should work well as configured, # however, to enhance performance for transfers involving large numbers of # files, you may experiment with hand tuning these values to optimize # performance for your particular system configuration. #parallel_process_count = %(parallel_process_count)d #parallel_thread_count = %(parallel_thread_count)d # 'parallel_composite_upload_threshold' specifies the maximum size of a file to # upload in a single stream. Files larger than this threshold will be # partitioned into component parts and uploaded in parallel and then composed # into a single object. # The number of components will be the smaller of # ceil(file_size / parallel_composite_upload_component_size) and # MAX_COMPOSE_ARITY. The current value of MAX_COMPOSE_ARITY is # %(max_compose_arity)d. # If 'parallel_composite_upload_threshold' is set to 0, then automatic parallel # uploads will never occur. # Setting an extremely low threshold is unadvisable. The vast majority of # environments will see degraded performance for thresholds below 80M, and it # is almost never advantageous to have a threshold below 20M. # 'parallel_composite_upload_component_size' specifies the ideal size of a # component in bytes, which will act as an upper bound to the size of the # components if ceil(file_size / parallel_composite_upload_component_size) is # less than MAX_COMPOSE_ARITY. # Values can be provided either in bytes or as human-readable values # (e.g., "150M" to represent 150 mebibytes) # # Note: At present parallel composite uploads are disabled by default, because # using composite objects requires a compiled crcmod (see "gsutil help crcmod"), # and for operating systems that don't already have this package installed this # makes gsutil harder to use. Google is actively working with a number of the # Linux distributions to get crcmod included with the stock distribution. Once # that is done we will re-enable parallel composite uploads by default in # gsutil. # # Note: Parallel composite uploads should not be used with NEARLINE, COLDLINE, # or ARCHIVE storage class buckets, as doing this incurs an early deletion # charge for each component object. # # Note: Parallel composite uploads are not enabled with Cloud KMS encrypted # objects as a source or destination, as composition with KMS objects is not yet # supported. #parallel_composite_upload_threshold = %(parallel_composite_upload_threshold)s #parallel_composite_upload_component_size = %(parallel_composite_upload_component_size)s # # 'parallel_composite_upload_bypass_kms_check' removes the object/bucket KMS checks # used to guard composition of KMS objects. #disable_parallel_composite_upload_kms_check = False # 'sliced_object_download_threshold' and # 'sliced_object_download_component_size' have analogous functionality to # their respective parallel_composite_upload config values. # 'sliced_object_download_max_components' specifies the maximum number of # slices to be used when performing a sliced object download. #sliced_object_download_threshold = %(sliced_object_download_threshold)s #sliced_object_download_component_size = %(sliced_object_download_component_size)s #sliced_object_download_max_components = %(sliced_object_download_max_components)s # Compressed transport encoded uploads buffer chunks of compressed data. When # running a composite upload and/or many uploads in parallel, compression may # consume more memory than available. This setting restricts the number of # compressed transport encoded uploads running in parallel such that they # don't consume more memory than set here. This is 2GiB by default. # Values can be provided either in bytes or as human-readable values # (e.g., "2G" to represent 2 gibibytes) #max_upload_compression_buffer_size = %(max_upload_compression_buffer_size)s # GZIP compression level, if using compression. Reducing this can have # a dramatic impact on compression speed with minor size increases. # This is a value from 0-9, with 9 being max compression. # A good level to try is 6, which is the default used by the gzip tool. #gzip_compression_level = %(gzip_compression_level)s # 'task_estimation_threshold' controls how many files or objects gsutil # processes before it attempts to estimate the total work that will be # performed by the command. Estimation makes extra directory listing or API # list calls and is performed only if multiple processes and/or threads are # used. Estimation can slightly increase cost due to extra # listing calls; to disable it entirely, set this value to 0. #task_estimation_threshold=%(task_estimation_threshold)s # 'use_magicfile' specifies if the 'file --mime <filename>' command should be # used to guess content types instead of the default filename extension-based # mechanism. Available on UNIX and macOS (and possibly on Windows, if you're # running Cygwin or some other package that provides implementations of # UNIX-like commands). When available and enabled use_magicfile should be more # robust because it analyzes file contents in addition to extensions. #use_magicfile = False # Service account emails for testing the hmac command. If these fields are not # populated with distinct service accounts the tests for the hmac command will # not be run. Primarily useful for tool developers. #test_hmac_service_account = #test_hmac_alt_service_account = #test_hmac_list_service_account = # Service account emails for testing impersonation credentials. If this field is # not populated with a service account the tests for service account # impersonation will not run. Primarily useful for tool developers. #test_impersonate_service_account = # 'content_language' specifies the ISO 639-1 language code of the content, to be # passed in the Content-Language header. By default no Content-Language is sent. # See the ISO 639-1 column of # http://www.loc.gov/standards/iso639-2/php/code_list.php for a list of # language codes. content_language = en # 'check_hashes' specifies how strictly to require integrity checking for # downloaded data. Legal values are: # '%(hash_fast_else_fail)s' - (default) Only integrity check if the digest # will run efficiently (using compiled code), else fail the download. # '%(hash_fast_else_skip)s' - Only integrity check if the server supplies a # hash and the local digest computation will run quickly, else skip the # check. # '%(hash_always)s' - Always check download integrity regardless of possible # performance costs. # '%(hash_never)s' - Don't perform download integrity checks. This setting is # not recommended except for special cases such as measuring download # performance excluding time for integrity checking. # This option exists to assist users who wish to download a GCS composite object # and are unable to install crcmod with the C-extension. CRC32c is the only # available integrity check for composite objects, and without the C-extension, # download performance can be significantly degraded by the digest computation. # This option is ignored for daisy-chain copies, which don't compute hashes but # instead (inexpensively) compare the cloud source and destination hashes. #check_hashes = if_fast_else_fail # 'encryption_key' specifies a single customer-supplied encryption key that # will be used for all data written to Google Cloud Storage. See # "gsutil help encryption" for more information # Encryption key: RFC 4648 section 4 base64-encoded AES256 string # Warning: If decrypt_key is specified without an encrypt_key, objects will be # decrypted when copied in the cloud. #encryption_key= # Each 'decryption_key' entry specifies a customer-supplied decryption key that # will be used to access and Google Cloud Storage objects encrypted with # the corresponding key. # Decryption keys: Up to 100 RFC 4648 section 4 base64-encoded AES256 strings # in ascending numerical order, starting with 1. #decryption_key1= #decryption_key2= #decryption_key3= # The ability to specify an alternative JSON API version is primarily for cloud # storage service developers. #json_api_version = v1 # Specifies the API to use when interacting with cloud storage providers. If the # gsutil command supports this API for the provider, it will be used instead of # the default API. Commands typically default to XML for S3 and JSON for GCS. # Note that if any encryption configuration options are set (see above), the # JSON API will be used for interacting with Google Cloud Storage buckets even # if XML is preferred, as gsutil does not currently support this functionality # when using the XML API. #prefer_api = json #prefer_api = xml # Disables the prompt asking for opt-in to data collection for analytics. #disable_analytics_prompt = True # The "test" command runs tests against regional buckets (unless you supply the # `-b` option). By default, the region used is us-central1, but you can change # the default region using this option. #test_cmd_regional_bucket_location = us-central1 # Tests for the "notification watchbucket" command require a notification URL. # If this option is not supplied, those tests will be skipped. #test_notification_url = https://yourdomain.url/notification-endpoint # Used in conjunction with --stet flag on cp command for end-to-end encryption. # STET binary path. If not specified, gsutil checks PATH for "stet". #stet_binary_path = <Path to binary "/usr/local/bin/stet"> # STET config path. If not specified, the STET binary will run with its default # settings. #stet_config_path = ~/.config/my_config.yaml # Adds an API call before parallel operations that triggers a reauth challenge. #trigger_reauth_challenge_for_parallel_operations = False """ % { 'hash_fast_else_fail': CHECK_HASH_IF_FAST_ELSE_FAIL, 'hash_fast_else_skip': CHECK_HASH_IF_FAST_ELSE_SKIP, 'hash_always': CHECK_HASH_ALWAYS, 'hash_never': CHECK_HASH_NEVER, 'resumable_threshold': constants.RESUMABLE_THRESHOLD_B, 'parallel_process_count': DEFAULT_PARALLEL_PROCESS_COUNT, 'parallel_thread_count': DEFAULT_PARALLEL_THREAD_COUNT, 'parallel_composite_upload_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'parallel_composite_upload_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_threshold': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD), 'sliced_object_download_component_size': (DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE), 'sliced_object_download_max_components': (DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS), 'max_compose_arity': MAX_COMPOSE_ARITY, 'task_estimation_threshold': DEFAULT_TASK_ESTIMATION_THRESHOLD, 'max_upload_compression_buffer_size': (DEFAULT_MAX_UPLOAD_COMPRESSION_BUFFER_SIZE), 'gzip_compression_level': DEFAULT_GZIP_COMPRESSION_LEVEL, } CONFIG_OAUTH2_CONFIG_CONTENT = """ [OAuth2] # This section specifies options used with OAuth2 authentication. # 'token_cache' specifies how the OAuth2 client should cache access tokens. # Valid values are: # 'in_memory': an in-memory cache is used. This is only useful if the boto # client instance (and with it the OAuth2 plugin instance) persists # across multiple requests. # 'file_system' : access tokens will be cached in the file system, in files # whose names include a key derived from the refresh token the access token # based on. # The default is 'file_system'. #token_cache = file_system #token_cache = in_memory # 'token_cache_path_pattern' specifies a path pattern for token cache files. # This option is only relevant if token_cache = file_system. # The value of this option should be a path, with place-holders '%(key)s' (which # will be replaced with a key derived from the refresh token the cached access # token was based on), and (optionally), %(uid)s (which will be replaced with # the UID of the current user, if available via os.getuid()). # Note that the config parser itself interpolates '%' placeholders, and hence # the above placeholders need to be escaped as '%%(key)s'. # The default value of this option is # token_cache_path_pattern = <tmpdir>/oauth2client-tokencache.%%(uid)s.%%(key)s # where <tmpdir> is the system-dependent default temp directory. # The following options specify the label and endpoint URIs for the OAUth2 # authorization provider being used. Primarily useful for tool developers. #provider_label = Google #provider_authorization_uri = https://accounts.google.com/o/oauth2/auth #provider_token_uri = https://oauth2.googleapis.com/token # 'oauth2_refresh_retries' controls the number of retry attempts made when # rate limiting errors occur for OAuth2 requests to retrieve an access token. # The default value is 6. #oauth2_refresh_retries = <integer value> # The following options specify the OAuth2 client identity and secret that is # used when requesting and using OAuth2 tokens. If not specified, a default # OAuth2 client for the gsutil tool is used; for uses of the boto library (with # OAuth2 authentication plugin) in other client software, it is recommended to # use a tool/client-specific OAuth2 client. For more information on OAuth2, see # http://code.google.com/apis/accounts/docs/OAuth2.html """ class ConfigCommand(Command): """Implementation of gsutil config command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'config', command_name_aliases=['cfg', 'conf', 'configure'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='abefhno:rs:w', supported_private_args=['reauth'], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='config', help_name_aliases=['cfg', 'conf', 'configure', 'aws', 's3'], help_type='command_help', help_one_line_summary=( 'Obtain credentials and create configuration file'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _OpenConfigFile(self, file_path): """Creates and opens a configuration file for writing. The file is created with mode 0600, and attempts to open existing files will fail (the latter is important to prevent symlink attacks). It is the caller's responsibility to close the file. Args: file_path: Path of the file to be created. Returns: A writable file object for the opened file. Raises: CommandException: if an error occurred when opening the file (including when the file already exists). """ flags = os.O_RDWR \| os.O_CREAT \| os.O_EXCL # Accommodate Windows; copied from python2.6/tempfile.py. if hasattr(os, 'O_NOINHERIT'): flags \|= os.O_NOINHERIT try: fd = os.open(file_path, flags, 0o600) except (OSError, IOError) as e: raise CommandException('Failed to open %s for writing: %s' % (file_path, e)) return os.fdopen(fd, 'w') def _CheckPrivateKeyFilePermissions(self, file_path): """Checks that the file has reasonable permissions for a private key. In particular, check that the filename provided by the user is not world- or group-readable. If either of these are true, we issue a warning and offer to fix the permissions. Args: file_path: The name of the private key file. """ if system_util.IS_WINDOWS: # For Windows, this check doesn't work (it actually just checks whether # the file is read-only). Since Windows files have a complicated ACL # system, this check doesn't make much sense on Windows anyway, so we # just don't do it. return st = os.stat(file_path) if bool((stat.S_IRGRP \| stat.S_IROTH) & st.st_mode): self.logger.warn( '\nYour private key file is readable by people other than yourself.\n' 'This is a security risk, since anyone with this information can use ' 'your service account.\n') fix_it = input('Would you like gsutil to change the file ' 'permissions for you? (y/N) ') if fix_it in ('y', 'Y'): try: os.chmod(file_path, 0o400) self.logger.info( '\nThe permissions on your file have been successfully ' 'modified.' '\nThe only access allowed is readability by the user ' '(permissions 0400 in chmod).') except Exception as _: # pylint: disable=broad-except self.logger.warn( '\nWe were unable to modify the permissions on your file.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') else: self.logger.info( '\nYou have chosen to allow this file to be readable by others.\n' 'If you would like to fix this yourself, consider running:\n' '"sudo chmod 400 </path/to/key>" for improved security.') def _WriteConfigLineMaybeCommented(self, config_file, name, value, desc): """Writes proxy name/value pair or comment line to config file. Writes proxy name/value pair if value is not None. Otherwise writes comment line. Args: config_file: File object to which the resulting config file will be written. name: The config variable name. value: The value, or None. desc: Human readable description (for comment). """ if not value: name = '#%s' % name value = '<%s>' % desc config_file.write('%s = %s\n' % (name, value)) def _WriteProxyConfigFileSection(self, config_file): """Writes proxy section of configuration file. Args: config_file: File object to which the resulting config file will be written. """ config = boto.config config_file.write( '# To use a proxy, edit and uncomment the proxy and proxy_port lines.\n' '# If you need a user/password with this proxy, edit and uncomment\n' '# those lines as well. If your organization also disallows DNS\n' '# lookups by client machines, set proxy_rdns to True (the default).\n' '# If you have installed gsutil through the Cloud SDK and have \n' '# configured proxy settings in gcloud, those proxy settings will \n' '# override any other options (including those set here, along with \n' '# any settings in proxy-related environment variables). Otherwise, \n' '# if proxy_host and proxy_port are not specified in this file and\n' '# one of the OS environment variables http_proxy, https_proxy, or\n' '# HTTPS_PROXY is defined, gsutil will use the proxy server specified\n' '# in these environment variables, in order of precedence according\n' '# to how they are listed above.\n') self._WriteConfigLineMaybeCommented(config_file, 'proxy', config.get_value('Boto', 'proxy', None), 'proxy host') self._WriteConfigLineMaybeCommented( config_file, 'proxy_type', config.get_value('Boto', 'proxy_type', None), 'proxy type (socks4, socks5, http) \| Defaults to http') self._WriteConfigLineMaybeCommented( config_file, 'proxy_port', config.get_value('Boto', 'proxy_port', None), 'proxy port') self._WriteConfigLineMaybeCommented( config_file, 'proxy_user', config.get_value('Boto', 'proxy_user', None), 'proxy user') self._WriteConfigLineMaybeCommented( config_file, 'proxy_pass', config.get_value('Boto', 'proxy_pass', None), 'proxy password') self._WriteConfigLineMaybeCommented( config_file, 'proxy_rdns', config.get_value('Boto', 'proxy_rdns', False), 'let proxy server perform DNS lookups (True,False); socks proxy not supported' ) # pylint: disable=dangerous-default-value,too-many-statements def _WriteBotoConfigFile(self, config_file, cred_type=CredTypes.OAUTH2_USER_ACCOUNT, configure_auth=True): """Creates a boto config file interactively. Needed credentials are obtained interactively, either by asking the user for access key and secret, or by walking the user through the OAuth2 approval flow. Args: config_file: File object to which the resulting config file will be written. cred_type: There are three options: - for HMAC, ask the user for access key and secret - for OAUTH2_USER_ACCOUNT, raise an error - for OAUTH2_SERVICE_ACCOUNT, prompt the user for OAuth2 for service account email address and private key file (and if the file is a .p12 file, the password for that file). configure_auth: Boolean, whether or not to configure authentication in the generated file. """ # Collect credentials provider_map = {'aws': 'aws', 'google': 'gs'} uri_map = {'aws': 's3', 'google': 'gs'} key_ids = {} sec_keys = {} service_account_key_is_json = False if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: gs_service_key_file = input('What is the full path to your private ' 'key file? ') # JSON files have the email address built-in and don't require a # password. try: with open(gs_service_key_file, 'rb') as key_file_fp: json.loads(key_file_fp.read()) service_account_key_is_json = True except ValueError: if not HAS_CRYPTO: raise CommandException( 'Service account authentication via a .p12 file requires ' 'either\nPyOpenSSL or PyCrypto 2.6 or later. Please install ' 'either of these\nto proceed, use a JSON-format key file, or ' 'configure a different type of credentials.') if not service_account_key_is_json: gs_service_client_id = input('What is your service account email ' 'address? ') gs_service_key_file_password = input('\n'.join( textwrap.wrap( 'What is the password for your service key file [if you ' 'haven\'t set one explicitly, leave this line blank]?')) + ' ') self._CheckPrivateKeyFilePermissions(gs_service_key_file) elif cred_type == CredTypes.OAUTH2_USER_ACCOUNT: raise CommandException( 'The user account authentication flow no longer works as of ' 'February 1, 2023. Tokens generated before this date will ' 'continue to work. To authenticate with your user account, ' 'install gsutil via Cloud SDK and run "gcloud auth login"') elif cred_type == CredTypes.HMAC: got_creds = False for provider in provider_map: if provider == 'google': key_ids[provider] = input('What is your %s access key ID? ' % provider) sec_keys[provider] = input('What is your %s secret access ' 'key? ' % provider) got_creds = True if not key_ids[provider] or not sec_keys[provider]: raise CommandException( 'Incomplete credentials provided. Please try again.') if not got_creds: raise CommandException('No credentials provided. Please try again.') # Write the config file prelude. config_file.write(CONFIG_PRELUDE_CONTENT.lstrip()) config_file.write( '# This file was created by gsutil version %s at %s.\n' % (gslib.VERSION, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))) config_file.write( '#\n# You can create additional configuration files by ' 'running\n# gsutil config [options] [-o <config-file>]\n\n\n') # Write the config file Credentials section. config_file.write('[Credentials]\n\n') if configure_auth: if cred_type == CredTypes.OAUTH2_SERVICE_ACCOUNT: config_file.write('# Google OAuth2 service account credentials ' '(for "gs://" URIs):\n') config_file.write('gs_service_key_file = %s\n' % gs_service_key_file) if not service_account_key_is_json: config_file.write('gs_service_client_id = %s\n' % gs_service_client_id) if not gs_service_key_file_password: config_file.write( '# If you would like to set your password, you can do so\n' '# using the following commands (replaced with your\n' '# information):\n' '# "openssl pkcs12 -in cert1.p12 -out temp_cert.pem"\n' '# "openssl pkcs12 -export -in temp_cert.pem -out cert2.p12"\n' '# "rm -f temp_cert.pem"\n' '# Your initial password is "notasecret" - for more\n' '# information, please see \n' '# http://www.openssl.org/docs/apps/pkcs12.html.\n') config_file.write('#gs_service_key_file_password =\n\n') else: config_file.write('gs_service_key_file_password = %s\n\n' % gs_service_key_file_password) else: config_file.write( '# To add Google OAuth2 credentials ("gs://" URIs), ' 'edit and uncomment the\n# following line:\n' '#gs_oauth2_refresh_token = <your OAuth2 refresh token>\n\n') else: if system_util.InvokedViaCloudSdk(): config_file.write( '# Google OAuth2 credentials are managed by the Cloud SDK and\n' '# do not need to be present in this file.\n') for provider in provider_map: key_prefix = provider_map[provider] uri_scheme = uri_map[provider] if provider in key_ids and provider in sec_keys: config_file.write('# %s credentials ("%s://" URIs):\n' % (provider, uri_scheme)) config_file.write('%s_access_key_id = %s\n' % (key_prefix, key_ids[provider])) config_file.write('%s_secret_access_key = %s\n' % (key_prefix, sec_keys[provider])) else: config_file.write( '# To add HMAC %s credentials for "%s://" URIs, edit and ' 'uncomment the\n# following two lines:\n' '#%s_access_key_id = <your %s access key ID>\n' '#%s_secret_access_key = <your %s secret access key>\n' % (provider, uri_scheme, key_prefix, provider, key_prefix, provider)) host_key = Provider.HostKeyMap[provider] config_file.write( '# The ability to specify an alternate storage host and port\n' '# is primarily for cloud storage service developers.\n' '# Setting a non-default gs_host only works if prefer_api=xml.\n' '#%s_host = <alternate storage host address>\n' '#%s_port = <alternate storage host port>\n' '# In some cases, (e.g. VPC requests) the "host" HTTP header should\n' '# be different than the host used in the request URL.\n' '#%s_host_header = <alternate storage host header>\n' % (host_key, host_key, host_key)) if host_key == 'gs': config_file.write( '#%s_json_host = <alternate JSON API storage host address>\n' '#%s_json_port = <alternate JSON API storage host port>\n' '#%s_json_host_header = <alternate JSON API storage host header>\n\n' % (host_key, host_key, host_key)) config_file.write( '# To impersonate a service account for "%s://" URIs over\n' '# JSON API, edit and uncomment the following line:\n' '#%s_impersonate_service_account = <service account email>\n\n') # Add device certificate mTLS fields. config_file.write( textwrap.dedent("""\ # This configuration setting enables or disables mutual TLS # authentication. The default value for this setting is "false". When # set to "true", gsutil uses the configured client certificate as # transport credential to access the APIs. The use of mTLS ensures that # the access originates from a trusted enterprise device. When enabled, # the client certificate is auto discovered using the endpoint # verification agent. When set to "true" but no client certificate or # key is found, users receive an error. #use_client_certificate = False # The command line to execute, which prints the # certificate, private key, or password to use in # conjunction with "use_client_certificate = True". #cert_provider_command = <Absolute path to command to run for # certification. Ex: "/scripts/gen_cert.sh"> """)) # Write the config file Boto section. config_file.write('%s\n' % CONFIG_BOTO_SECTION_CONTENT) self._WriteProxyConfigFileSection(config_file) # Write the Google Compute Engine section. config_file.write(CONFIG_GOOGLECOMPUTE_SECTION_CONTENT) # Write the config file GSUtil section that doesn't depend on user input. config_file.write(CONFIG_INPUTLESS_GSUTIL_SECTION_CONTENT) # Write the default API version. config_file.write(""" # 'default_api_version' specifies the default Google Cloud Storage XML API # version to use. If not set below gsutil defaults to API version 1. """) api_version = 2 if cred_type == CredTypes.HMAC: api_version = 1 config_file.write('default_api_version = %d\n' % api_version) # Write the config file GSUtil section that includes the default # project ID input from the user. if not system_util.InvokedViaCloudSdk(): default_project_id = input('What is your project-id? ').strip() project_id_section_prelude = """ # 'default_project_id' specifies the default Google Cloud Storage project ID to # use with the 'mb' and 'ls' commands. This default can be overridden by # specifying the -p option to the 'mb' and 'ls' commands. """ if not default_project_id: raise CommandException( 'No default project ID entered. The default project ID is needed ' 'by the\nls and mb commands; please try again.') config_file.write('%sdefault_project_id = %s\n\n\n' % (project_id_section_prelude, default_project_id)) CheckAndMaybePromptForAnalyticsEnabling() # Write the config file OAuth2 section that doesn't depend on user input. config_file.write(CONFIG_OAUTH2_CONFIG_CONTENT) config_file.write('#client_id = <OAuth2 client id>\n' '#client_secret = <OAuth2 client secret>\n') def RunCommand(self): """Command entry point for the config command.""" cred_type = CredTypes.OAUTH2_USER_ACCOUNT output_file_name = None has_a = False has_e = False configure_auth = True for opt, opt_arg in self.sub_opts: if opt == '-a': cred_type = CredTypes.HMAC has_a = True elif opt == '-e': cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT has_e = True elif opt == '-n': configure_auth = False elif opt == '-o': output_file_name = opt_arg else: self.RaiseInvalidArgumentException() if has_e and has_a: raise CommandException('Both -a and -e cannot be specified. Please see ' '"gsutil help config" for more information.') if not configure_auth and (has_a or has_e): raise CommandException( 'The -a and -e flags cannot be specified with the -n flag. Please ' 'see "gsutil help config" for more information.') # Don't allow users to configure Oauth2 (any option other than -a and -n) # when running in the Cloud SDK, unless they have the Cloud SDK configured # not to pass credentials to gsutil. if (system_util.InvokedViaCloudSdk() and system_util.CloudSdkCredPassingEnabled() and not has_a and configure_auth): raise CommandException('\n'.join([ 'OAuth2 is the preferred authentication mechanism with the Cloud ' 'SDK.', 'Run "gcloud auth login" to configure authentication, unless:', '\n'.join( textwrap.wrap( 'You don\'t want gsutil to use OAuth2 credentials from the Cloud ' 'SDK, but instead want to manage credentials with .boto files ' 'generated by running "gsutil config"; in which case run ' '"gcloud config set pass_credentials_to_gsutil false".', initial_indent='- ', subsequent_indent=' ')), '\n'.join( textwrap.wrap( 'You want to authenticate with an HMAC access key and secret, in ' 'which case run "gsutil config -a".', initial_indent='- ', subsequent_indent=' ')), ])) if system_util.InvokedViaCloudSdk() and has_a: sys.stderr.write('\n'.join( textwrap.wrap( 'This command will configure HMAC credentials, but gsutil will use ' 'OAuth2 credentials from the Cloud SDK by default. To make sure ' 'the HMAC credentials are used, run: "gcloud config set ' 'pass_credentials_to_gsutil false".')) + '\n\n') default_config_path_bak = None if not output_file_name: # Check to see if a default config file name is requested via # environment variable. If so, use it, otherwise use the hard-coded # default file. Then use the default config file name, if it doesn't # exist or can be moved out of the way without clobbering an existing # backup file. boto_config_from_env = os.environ.get('BOTO_CONFIG', None) if boto_config_from_env: default_config_path = boto_config_from_env else: default_config_path = os.path.expanduser(os.path.join('~', '.boto')) if not os.path.exists(default_config_path): output_file_name = default_config_path else: default_config_path_bak = default_config_path + '.bak' if os.path.exists(default_config_path_bak): raise CommandException('Cannot back up existing config ' 'file "%s": backup file exists ("%s").' % (default_config_path, default_config_path_bak)) else: try: sys.stderr.write( 'Backing up existing config file "%s" to "%s"...\n' % (default_config_path, default_config_path_bak)) os.rename(default_config_path, default_config_path_bak) except Exception as e: raise CommandException( 'Failed to back up existing config ' 'file ("%s" -> "%s"): %s.' % (default_config_path, default_config_path_bak, e)) output_file_name = default_config_path if output_file_name == '-': output_file = sys.stdout else: output_file = self._OpenConfigFile(output_file_name) sys.stderr.write('\n'.join( textwrap.wrap( 'This command will create a boto config file at %s containing your ' 'credentials, based on your responses to the following questions.' % output_file_name)) + '\n') # Catch ^C so we can restore the backup. RegisterSignalHandler(signal.SIGINT, _CleanupHandler) try: self._WriteBotoConfigFile(output_file, cred_type=cred_type, configure_auth=configure_auth) except Exception as e: user_aborted = isinstance(e, AbortException) if user_aborted: sys.stderr.write('\nCaught ^C; cleaning up\n') # If an error occurred during config file creation, remove the invalid # config file and restore the backup file. if output_file_name != '-': output_file.close() os.unlink(output_file_name) try: if default_config_path_bak: sys.stderr.write('Restoring previous backed up file (%s)\n' % default_config_path_bak) os.rename(default_config_path_bak, output_file_name) except Exception as e: # Raise the original exception so that we can see what actually went # wrong, rather than just finding out that we died before assigning # a value to default_config_path_bak. raise e raise if output_file_name != '-': output_file.close() if not boto.config.has_option('Boto', 'proxy'): sys.stderr.write('\n' + '\n'.join( textwrap.wrap( 'Boto config file "%s" created.\nIf you need to use a proxy to ' 'access the Internet please see the instructions in that file.' % output_file_name)) + '\n') return 0 def _CleanupHandler(unused_signalnum, unused_handler): raise AbortException('User interrupted config command')	51.14 KB	2025-06-25 08:19
📄 cors.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of cors configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import CorsTranslation from gslib.utils.translation_helper import REMOVE_CORS_CONFIG _GET_SYNOPSIS = """ gsutil cors get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil cors set <cors-json-file> gs://<bucket_name>... """ _GET_DESCRIPTION = """ <B>GET</B> Gets the CORS configuration for a single bucket. The output from ``cors get`` can be redirected into a file, edited and then updated using ``cors set``. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the CORS configuration for one or more buckets. The ``cors-json-file`` specified on the command line should be a path to a local file containing a JSON-formatted CORS configuration, such as the example described above. """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n\n' _DESCRIPTION = (""" Gets or sets the Cross-Origin Resource Sharing (CORS) configuration on one or more buckets. This command is supported for buckets only, not objects. An example CORS JSON file looks like the following: [ { "origin": ["http://origin1.example.com"], "responseHeader": ["Content-Type"], "method": ["GET"], "maxAgeSeconds": 3600 } ] The above CORS configuration explicitly allows cross-origin GET requests from http://origin1.example.com and may include the Content-Type response header. The preflight request may be cached for 1 hour. Note that requests to the authenticated browser download endpoint ``storage.cloud.google.com`` do not allow CORS requests. For more information about supported endpoints for CORS, see `Cloud Storage CORS support <https://cloud.google.com/storage/docs/cross-origin#server-side-support>`_. The following (empty) CORS JSON file removes any CORS configuration for a bucket: [] The cors command has two sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION]) + """ For more info about CORS generally, see https://www.w3.org/TR/cors/. For more info about CORS in Cloud Storage, see the `CORS concept page <https://cloud.google.com/storage/docs/cross-origin>`_. """) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class CorsCommand(Command): """Implementation of gsutil cors command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cors', command_name_aliases=['getcors', 'setcors'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cors', help_name_aliases=[ 'getcors', 'setcors', 'cross-origin', ], help_type='command_help', help_one_line_summary=( 'Get or set a CORS configuration for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=cors_config,empty=\' has no ' 'CORS configuration.\',empty_prefix_key=storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--cors-file' ], flag_map={}, ), }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetCors(self): """Sets CORS configuration on a Google Cloud Storage bucket.""" cors_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'cors set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. cors_file = open(cors_arg, 'r') cors_txt = cors_file.read() cors_file.close() self.api = self.gsutil_api.GetApiSelector( StorageUrlFromString(url_args[0]).scheme) # Iterate over URLs, expanding wildcards and setting the CORS on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting CORS on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetCors(cors_txt, url, provider=url.scheme) else: cors = CorsTranslation.JsonCorsToMessageEntries(cors_txt) if not cors: cors = REMOVE_CORS_CONFIG bucket_metadata = apitools_messages.Bucket(cors=cors) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetCors(self): """Gets CORS configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['cors']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetCors(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.cors: sys.stdout.write( CorsTranslation.MessageEntriesToJson(bucket_metadata.cors)) else: sys.stdout.write('%s has no CORS configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the cors command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetCors elif action_subcommand == 'set': func = self._SetCors else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help cors".') % (action_subcommand, self.command_name)) metrics.LogCommandParams(subcommands=[action_subcommand]) return func()	8.54 KB	2025-06-25 08:19
📄 cp.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like cp command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import itertools import logging import os import time import traceback from apitools.base.py import encoding from gslib import gcs_json_api from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import DestinationInfo from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import NameExpansionIteratorDestinationTuple from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import ContainsWildcard from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import cat_helper from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS from gslib.utils.constants import NO_MAX from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import ItemExistsError from gslib.utils.copy_helper import Manifest from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import GetStreamFromFileUrl from gslib.utils.system_util import StdinIterator from gslib.utils.system_util import StdinIteratorCls from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.text_util import RemoveCRLFFromString from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import MakeHumanReadable _SYNOPSIS = """ gsutil cp [OPTION]... src_url dst_url gsutil cp [OPTION]... src_url... dst_url gsutil cp [OPTION]... -I dst_url """ _SYNOPSIS_TEXT = """ <B>SYNOPSIS</B> """ + _SYNOPSIS _DESCRIPTION_TEXT = """ <B>DESCRIPTION</B> The ``gsutil cp`` command allows you to copy data between your local file system and the cloud, within the cloud, and between cloud storage providers. For example, to upload all text files from the local directory to a bucket, you can run: gsutil cp .txt gs://my-bucket You can also download data from a bucket. The following command downloads all text files from the top-level of a bucket to your current directory: gsutil cp gs://my-bucket/.txt . You can use the ``-n`` option to prevent overwriting the content of existing files. The following example downloads text files from a bucket without clobbering the data in your directory: gsutil cp -n gs://my-bucket/.txt . Use the ``-r`` option to copy an entire directory tree. For example, to upload the directory tree ``dir``: gsutil cp -r dir gs://my-bucket If you have a large number of files to transfer, you can perform a parallel multi-threaded/multi-processing copy using the top-level gsutil ``-m`` option (see "gsutil help options"): gsutil -m cp -r dir gs://my-bucket You can use the ``-I`` option with ``stdin`` to specify a list of URLs to copy, one per line. This allows you to use gsutil in a pipeline to upload or download objects as generated by a program: cat filelist \| gsutil -m cp -I gs://my-bucket or: cat filelist \| gsutil -m cp -I ./download_dir where the output of ``cat filelist`` is a list of files, cloud URLs, and wildcards of files and cloud URLs. NOTE: Shells like ``bash`` and ``zsh`` sometimes attempt to expand wildcards in ways that can be surprising. You may also encounter issues when attempting to copy files whose names contain wildcard characters. For more details about these issues, see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. """ _NAME_CONSTRUCTION_TEXT = """ <B>HOW NAMES ARE CONSTRUCTED</B> The ``gsutil cp`` command attempts to name objects in ways that are consistent with the Linux ``cp`` command. This means that names are constructed depending on whether you're performing a recursive directory copy or copying individually-named objects, or whether you're copying to an existing or non-existent directory. When you perform recursive directory copies, object names are constructed to mirror the source directory structure starting at the point of recursive processing. For example, if ``dir1/dir2`` contains the file ``a/b/c``, then the following command creates the object ``gs://my-bucket/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket In contrast, copying individually-named files results in objects named by the final path component of the source files. For example, assuming again that ``dir1/dir2`` contains ``a/b/c``, the following command creates the object ``gs://my-bucket/c``: gsutil cp dir1/dir2/* gs://my-bucket Note that in the above example, the '*' wildcard matches all names anywhere under ``dir``. The wildcard '' matches names just one level deep. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. The same rules apply for uploads and downloads: recursive copies of buckets and bucket subdirectories produce a mirrored filename structure, while copying individually or wildcard-named objects produce flatly-named files. In addition, the resulting names depend on whether the destination subdirectory exists. For example, if ``gs://my-bucket/subdir`` exists as a subdirectory, the following command creates the object ``gs://my-bucket/subdir/dir2/a/b/c``: gsutil cp -r dir1/dir2 gs://my-bucket/subdir In contrast, if ``gs://my-bucket/subdir`` does not exist, this same ``gsutil cp`` command creates the object ``gs://my-bucket/subdir/a/b/c``. NOTE: The `Google Cloud Platform Console <https://console.cloud.google.com>`_ creates folders by creating "placeholder" objects that end with a "/" character. gsutil skips these objects when downloading from the cloud to the local file system, because creating a file that ends with a "/" is not allowed on Linux and macOS. We recommend that you only create objects that end with "/" if you don't intend to download such objects using gsutil. """ _SUBDIRECTORIES_TEXT = """ <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B> You can use gsutil to copy to and from subdirectories by using a command like this: gsutil cp -r dir gs://my-bucket/data This causes ``dir`` and all of its files and nested subdirectories to be copied under the specified destination, resulting in objects with names like ``gs://my-bucket/data/dir/a/b/c``. Similarly, you can download from bucket subdirectories using the following command: gsutil cp -r gs://my-bucket/data dir This causes everything nested under ``gs://my-bucket/data`` to be downloaded into ``dir``, resulting in files with names like ``dir/data/a/b/c``. Copying subdirectories is useful if you want to add data to an existing bucket directory structure over time. It's also useful if you want to parallelize uploads and downloads across multiple machines (potentially reducing overall transfer time compared with running ``gsutil -m cp`` on one machine). For example, if your bucket contains this structure: gs://my-bucket/data/result_set_01/ gs://my-bucket/data/result_set_02/ ... gs://my-bucket/data/result_set_99/ you can perform concurrent downloads across 3 machines by running these commands on each machine, respectively: gsutil -m cp -r gs://my-bucket/data/result_set_[0-3]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[4-6]* dir gsutil -m cp -r gs://my-bucket/data/result_set_[7-9]* dir Note that ``dir`` could be a local directory on each machine, or a directory mounted off of a shared file server. The performance of the latter depends on several factors, so we recommend experimenting to find out what works best for your computing environment. """ _COPY_IN_CLOUD_TEXT = """ <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata such as ``Content-Type`` and ``Cache-Control``. In contrast, when you download data from the cloud, it ends up in a file with no associated metadata, unless you have some way to keep or re-create that metadata. Copies spanning locations and/or storage classes cause data to be rewritten in the cloud, which may take some time (but is still faster than downloading and re-uploading). Such operations can be resumed with the same command if they are interrupted, so long as the command parameters are identical. Note that by default, the gsutil ``cp`` command does not copy the object ACL to the new object, and instead uses the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the ``-p`` option. When copying in the cloud, if the destination bucket has Object Versioning enabled, by default ``gsutil cp`` copies only live versions of the source object. For example, the following command causes only the single live version of ``gs://bucket1/obj`` to be copied to ``gs://bucket2``, even if there are noncurrent versions of ``gs://bucket1/obj``: gsutil cp gs://bucket1/obj gs://bucket2 To also copy noncurrent versions, use the ``-A`` flag: gsutil cp -A gs://bucket1/obj gs://bucket2 The top-level gsutil ``-m`` flag is not allowed when using the ``cp -A`` flag. """ _CHECKSUM_VALIDATION_TEXT = """ <B>CHECKSUM VALIDATION</B> gsutil automatically performs checksum validation for copies to and from Cloud Storage. For more information, see `Hashes and ETags <https://cloud.google.com/storage/docs/hashes-etags#cli>`_. """ _RETRY_HANDLING_TEXT = """ <B>RETRY HANDLING</B> The ``cp`` command retries when failures occur, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the ``cp`` command skips that object and moves on. If any failures were not successfully retried by the end of the copy run, the ``cp`` command reports the number of failures and exits with a non-zero status. For details about gsutil's overall retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. """ _RESUMABLE_TRANSFERS_TEXT = """ <B>RESUMABLE TRANSFERS</B> gsutil automatically resumes interrupted downloads and interrupted `resumable uploads <https://cloud.google.com/storage/docs/resumable-uploads#gsutil>`_, except when performing streaming transfers. In the case of an interrupted download, a partially downloaded temporary file is visible in the destination directory with the suffix ``_.gstmp`` in its name. Upon completion, the original file is deleted and replaced with the downloaded contents. Resumable transfers store state information in files under ~/.gsutil, named by the destination object or file. See "gsutil help prod" for details on using resumable transfers in production. """ _STREAMING_TRANSFERS_TEXT = """ <B>STREAMING TRANSFERS</B> Use '-' in place of src_url or dst_url to perform a `streaming transfer <https://cloud.google.com/storage/docs/streaming>`_. Streaming uploads using the `JSON API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ are buffered in memory part-way back into the file and can thus sometimes resume in the event of network or service problems. gsutil does not support resuming streaming uploads using the XML API or resuming streaming downloads for either JSON or XML. If you have a large amount of data to transfer in these cases, we recommend that you write the data to a local file and copy that file rather than streaming it. """ _SLICED_OBJECT_DOWNLOADS_TEXT = """ <B>SLICED OBJECT DOWNLOADS</B> gsutil can automatically use ranged ``GET`` requests to perform downloads in parallel for large files being downloaded from Cloud Storage. See `sliced object download documentation <https://cloud.google.com/storage/docs/sliced-object-downloads>`_ for a complete discussion. """ _PARALLEL_COMPOSITE_UPLOADS_TEXT = """ <B>PARALLEL COMPOSITE UPLOADS</B> gsutil can automatically use `object composition <https://cloud.google.com/storage/docs/composite-objects>`_ to perform uploads in parallel for large, local files being uploaded to Cloud Storage. See the `parallel composite uploads documentation <https://cloud.google.com/storage/docs/parallel-composite-uploads>`_ for a complete discussion. """ _CHANGING_TEMP_DIRECTORIES_TEXT = """ <B>CHANGING TEMP DIRECTORIES</B> gsutil writes data to a temporary directory in several cases: - when compressing data to be uploaded (see the ``-z`` and ``-Z`` options) - when decompressing data being downloaded (for example, when the data has ``Content-Encoding:gzip`` as a result of being uploaded using gsutil cp -z or gsutil cp -Z) - when running integration tests using the gsutil test command In these cases, it's possible the temporary file location on your system that gsutil selects by default may not have enough space. If gsutil runs out of space during one of these operations (for example, raising "CommandException: Inadequate temp space available to compress <your file>" during a ``gsutil cp -z`` operation), you can change where it writes these temp files by setting the TMPDIR environment variable. On Linux and macOS, you can set the variable as follows: TMPDIR=/some/directory gsutil cp ... You can also add this line to your ~/.bashrc file and restart the shell before running gsutil: export TMPDIR=/some/directory On Windows 7, you can change the TMPDIR environment variable from Start -> Computer -> System -> Advanced System Settings -> Environment Variables. You need to reboot after making this change for it to take effect. Rebooting is not necessary after running the export command on Linux and macOS. """ _COPYING_SPECIAL_FILES_TEXT = """ <B>SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SUCH AS SYMLINKS AND DEVICES)</B> Please see the section about OS-specific file types in "gsutil help rsync". While that section refers to the ``rsync`` command, analogous points apply to the ``cp`` command. """ _OPTIONS_TEXT = """ <B>OPTIONS</B> -a predef_acl Applies the specific predefined ACL to uploaded objects. See "gsutil help acls" for further details. -A Copy all source versions from a source bucket or folder. If not set, only the live version of each source object is copied. NOTE: This option is only useful when the destination bucket has Object Versioning enabled. Additionally, the generation numbers of copied versions do not necessarily match the order of the original generation numbers. -c If an error occurs, continue attempting to copy the remaining files. If any copies are unsuccessful, gsutil's exit status is non-zero, even if this flag is set. This option is implicitly set when running ``gsutil -m cp...``. NOTE: ``-c`` only applies to the actual copying operation. If an error, such as ``invalid Unicode file name``, occurs while iterating over the files in the local directory, gsutil prints an error message and aborts. -D Copy in "daisy chain" mode, which means copying between two buckets by first downloading to the machine where gsutil is run, then uploading to the destination bucket. The default mode is a "copy in the cloud," where data is copied between two buckets without uploading or downloading. During a "copy in the cloud," a source composite object remains composite at its destination. However, you can use "daisy chain" mode to change a composite object into a non-composite object. For example: gsutil cp -D gs://bucket/obj gs://bucket/obj_tmp gsutil mv gs://bucket/obj_tmp gs://bucket/obj NOTE: "Daisy chain" mode is automatically used when copying between providers: for example, when copying data from Cloud Storage to another provider. -e Exclude symlinks. When specified, symbolic links are not copied. -I Use ``stdin`` to specify a list of files or objects to copy. You can use gsutil in a pipeline to upload or download objects as generated by a program. For example: cat filelist \| gsutil -m cp -I gs://my-bucket where the output of ``cat filelist`` is a one-per-line list of files, cloud URLs, and wildcards of files and cloud URLs. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the ``-j`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files. This also saves network bandwidth while leaving the data uncompressed in Cloud Storage. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the ``Content-Type`` and name of the original files. Note that if you want to use the ``-m`` `top-level option <https://cloud.google.com/storage/docs/gsutil/addlhelp/GlobalCommandLineOptions>`_ to parallelize copies along with the ``-j/-J`` options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. You can change this compression buffer size to a higher limit. For example: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m cp -j html,txt -r /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the ``-j`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in longer uploads. -L <file> Outputs a manifest log file with detailed information about each item that was copied. This manifest contains the following information for each item: - Source path. - Destination path. - Source size. - Bytes transferred. - MD5 hash. - Transfer start time and date in UTC and ISO 8601 format. - Transfer completion time and date in UTC and ISO 8601 format. - Upload id, if a resumable upload was performed. - Final result of the attempted transfer, either success or failure. - Failure details, if any. If the log file already exists, gsutil uses the file as an input to the copy process, and appends log items to the existing file. Objects that are marked in the existing log file as having been successfully copied or skipped are ignored. Objects without entries are copied and ones previously marked as unsuccessful are retried. This option can be used in conjunction with the ``-c`` option to build a script that copies a large number of objects reliably, using a bash script like the following: until gsutil cp -c -L cp.log -r ./dir gs://bucket; do sleep 1 done The -c option enables copying to continue after failures occur, and the -L option allows gsutil to pick up where it left off without duplicating work. The loop continues running as long as gsutil exits with a non-zero status. A non-zero status indicates there was at least one failure during the copy operation. NOTE: If you are synchronizing the contents of a directory and a bucket, or the contents of two buckets, see "gsutil help rsync". -n No-clobber. When specified, existing files or objects at the destination are not replaced. Any items that are skipped by this option are reported as skipped. gsutil performs an additional GET request to check if an item exists before attempting to upload the data. This saves gsutil from retransmitting data, but the additional HTTP requests may make small object transfers slower and more expensive. -p Preserves ACLs when copying in the cloud. Note that this option has performance and cost implications only when using the XML API, as the XML API requires separate HTTP calls for interacting with ACLs. You can mitigate this performance issue using ``gsutil -m cp`` to perform parallel copying. Note that this option only works if you have OWNER access to all objects that are copied. If you want all objects in the destination bucket to end up with the same ACL, you can avoid these performance issues by setting a default object ACL on that bucket instead of using ``cp -p``. See "gsutil help defacl". Note that it's not valid to specify both the ``-a`` and ``-p`` options together. -P Enables POSIX attributes to be preserved when objects are copied. ``gsutil cp`` copies fields provided by ``stat``. These fields are the user ID of the owner, the group ID of the owning group, the mode or permissions of the file, and the access and modification time of the file. For downloads, these attributes are only set if the source objects were uploaded with this flag enabled. On Windows, this flag only sets and restores access time and modification time. This is because Windows doesn't support POSIX uid/gid/mode. -R, -r The ``-R`` and ``-r`` options are synonymous. They enable directories, buckets, and bucket subdirectories to be copied recursively. If you don't use this option for an upload, gsutil copies objects it finds and skips directories. Similarly, if you don't specify this option for a download, gsutil copies objects at the current bucket directory level and skips subdirectories. -s <class> Specifies the storage class of the destination object. If not specified, the default storage class of the destination bucket is used. This option is not valid for copying to non-cloud destinations. -U Skips objects with unsupported object types instead of failing. Unsupported object types include Amazon S3 objects in the GLACIER storage class. -v Prints the version-specific URL for each uploaded object. You can use these URLs to safely make concurrent upload requests, because Cloud Storage refuses to perform an update if the current object version doesn't match the version-specific URL. See `generation numbers <https://cloud.google.com/storage/docs/metadata#generation-number>`_ for more details. -z <ext,...> Applies gzip content-encoding to any file upload whose extension matches the ``-z`` extension list. This is useful when uploading files with compressible content such as .js, .css, or .html files, because it reduces network bandwidth and storage sizes. This can both improve performance and reduce costs. When you specify the ``-z`` option, the data from your files is compressed before it is uploaded, but your actual files are left uncompressed on the local disk. The uploaded objects retain the ``Content-Type`` and name of the original files, but have their ``Content-Encoding`` metadata set to ``gzip`` to indicate that the object data stored are compressed on the Cloud Storage servers and have their ``Cache-Control`` metadata set to ``no-transform``. For example, the following command: gsutil cp -z html \\ cattypes.html tabby.jpeg gs://mycats does the following: - The ``cp`` command uploads the files ``cattypes.html`` and ``tabby.jpeg`` to the bucket ``gs://mycats``. - Based on the file extensions, gsutil sets the ``Content-Type`` of ``cattypes.html`` to ``text/html`` and ``tabby.jpeg`` to ``image/jpeg``. - The ``-z`` option compresses the data in the file ``cattypes.html``. - The ``-z`` option also sets the ``Content-Encoding`` for ``cattypes.html`` to ``gzip`` and the ``Cache-Control`` for ``cattypes.html`` to ``no-transform``. Because the ``-z/-Z`` options compress data prior to upload, they are not subject to the same compression buffer bottleneck that can affect the ``-j/-J`` options. Note that if you download an object with ``Content-Encoding:gzip``, gsutil decompresses the content before writing the local file. -Z Applies gzip content-encoding to file uploads. This option works like the ``-z`` option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If some of the source files don't compress well, such as binary data, using this option may result in files taking up more space in the cloud than they would if left uncompressed. --stet If the STET binary can be found in boto or PATH, cp will use the split-trust encryption tool for end-to-end encryption. """ _DETAILED_HELP_TEXT = '\n\n'.join([ _SYNOPSIS_TEXT, _DESCRIPTION_TEXT, _NAME_CONSTRUCTION_TEXT, _SUBDIRECTORIES_TEXT, _COPY_IN_CLOUD_TEXT, _CHECKSUM_VALIDATION_TEXT, _RETRY_HANDLING_TEXT, _RESUMABLE_TRANSFERS_TEXT, _STREAMING_TRANSFERS_TEXT, _SLICED_OBJECT_DOWNLOADS_TEXT, _PARALLEL_COMPOSITE_UPLOADS_TEXT, _CHANGING_TEMP_DIRECTORIES_TEXT, _COPYING_SPECIAL_FILES_TEXT, _OPTIONS_TEXT, ]) CP_SUB_ARGS = 'a:AcDeIL:MNnpPrRs:tUvz:Zj:J' # May be used by cp or mv. CP_AND_MV_SHIM_FLAG_MAP = { '-A': GcloudStorageFlag('--all-versions'), '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--continue-on-error'), '-D': GcloudStorageFlag('--daisy-chain'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-I': GcloudStorageFlag('--read-paths-from-stdin'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-L': GcloudStorageFlag('--manifest-path'), '-n': GcloudStorageFlag('--no-clobber'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-s': GcloudStorageFlag('--storage-class'), '-v': GcloudStorageFlag('--print-created-message'), '-Z': GcloudStorageFlag('--gzip-local-all'), '-z': GcloudStorageFlag('--gzip-local'), '-U': GcloudStorageFlag('--skip-unsupported'), } # Adds recursion flags. CP_SHIM_FLAG_MAP = { k: v for k, v in list(CP_AND_MV_SHIM_FLAG_MAP.items()) + [('-r', GcloudStorageFlag('-r')), ('-R', GcloudStorageFlag('-r'))] } def ShimTranslatePredefinedAclSubOptForCopy(sub_opts): """Gcloud uses camel-case predefined/canned ACLs, and gsutil uses snake-case. The camel-case-snake-case difference is related to gcloud primarily using JSON API rather than the XML API. Predefined ACLs are also called "canned ACLs". Args: sub_opts: List of pairs representing flag keys and values, e.g. [('a', 'public-read')] """ predefined_acl_idx = None for i, (k, _) in enumerate(sub_opts): if k == '-a': predefined_acl_idx = i break if predefined_acl_idx is not None: old_predefined_acl = sub_opts[i][1] sub_opts[i] = (sub_opts[i][0], gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.get( old_predefined_acl, old_predefined_acl)) def _CopyFuncWrapper(cls, args, thread_state=None): cls.CopyFunc(args, thread_state=thread_state, preserve_posix=cls.preserve_posix_attrs) def _CopyExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while copying:\n%s\n', traceback.format_exc()) def _RmExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) class CpCommand(Command): """Implementation of gsutil cp command. Note that CpCommand is run for both gsutil cp and gsutil mv. The latter happens by MvCommand calling CpCommand and passing the hidden (undocumented) -M option. This allows the copy and remove needed for each mv to run together (rather than first running all the cp's and then all the rm's, as we originally had implemented), which in turn avoids the following problem with removing the wrong objects: starting with a bucket containing only the object gs://bucket/obj, say the user does: gsutil mv gs://bucket/* gs://bucket/d.txt If we ran all the cp's and then all the rm's and we didn't expand the wildcard first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt, and the rm command would then remove that object. In the implementation prior to gsutil release 3.12 we avoided this by building a list of objects to process and then running the copies and then the removes; but building the list up front limits scalability (compared with the current approach of processing the bucket listing iterator on the fly). """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'cp', command_name_aliases=['copy'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # -t is deprecated but leave intact for now to avoid breakage. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, # Unfortunately, "private" args are the only way to support non-single # character flags. supported_private_args=['stet', 'testcallbackfile='], argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='cp', help_name_aliases=['copy'], help_type='command_help', help_one_line_summary='Copy files and objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): self.logger.warn( "Unlike pure gsutil, this shim won't run composite uploads and sliced" ' downloads in parallel by default. Use the -m flag to enable' ' parallelism (i.e. "gsutil -m cp ...").') ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'cp'], flag_map=CP_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) # pylint: disable=too-many-statements def CopyFunc(self, copy_object_info, thread_state=None, preserve_posix=False): """Worker function for performing the actual copy (and rm, for mv).""" gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) copy_helper_opts = copy_helper.GetCopyHelperOpts() if copy_helper_opts.perform_mv: cmd_name = 'mv' else: cmd_name = self.command_name src_url = copy_object_info.source_storage_url exp_src_url = copy_object_info.expanded_storage_url src_url_names_container = copy_object_info.names_container have_multiple_srcs = copy_object_info.is_multi_source_request if src_url.IsCloudUrl() and src_url.IsProvider(): raise CommandException( 'The %s command does not allow provider-only source URLs (%s)' % (cmd_name, src_url)) if preserve_posix and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException('Cannot preserve POSIX attributes with a stream.') if self.parallel_operations and src_url.IsFileUrl() and src_url.IsStream(): raise CommandException( 'Cannot upload from a stream when using gsutil -m option.') if have_multiple_srcs: copy_helper.InsistDstUrlNamesContainer( copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, cmd_name) # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. We # need to filter them out here, otherwise if the user tries to rsync from # GCS to a local directory it will result in a directory/file conflict # (e.g., trying to download an object called "mydata/" where the local # directory "mydata" exists). if IsCloudSubdirPlaceholder(exp_src_url): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. return if copy_helper_opts.use_manifest and self.manifest.WasSuccessful( exp_src_url.url_string): return if copy_helper_opts.perform_mv and copy_object_info.names_container: # Use recursion_requested when performing name expansion for the # directory mv case so we can determine if any of the source URLs are # directories (and then use cp -r and rm -r to perform the move, to # match the behavior of Linux mv (which when moving a directory moves # all the contained files). self.recursion_requested = True if (copy_object_info.exp_dst_url.IsFileUrl() and not os.path.exists(copy_object_info.exp_dst_url.object_name) and have_multiple_srcs): try: os.makedirs(copy_object_info.exp_dst_url.object_name) except OSError as e: if e.errno != errno.EEXIST: raise dst_url = copy_helper.ConstructDstUrl( src_url, exp_src_url, src_url_names_container, have_multiple_srcs, copy_object_info.is_multi_top_level_source_request, copy_object_info.exp_dst_url, copy_object_info.have_existing_dst_container, self.recursion_requested, preserve_posix=preserve_posix) dst_url = copy_helper.FixWindowsNaming(src_url, dst_url) copy_helper.CheckForDirFileConflict(exp_src_url, dst_url) if copy_helper.SrcDstSame(exp_src_url, dst_url): raise CommandException('%s: "%s" and "%s" are the same file - ' 'abort.' % (cmd_name, exp_src_url, dst_url)) if dst_url.IsCloudUrl() and dst_url.HasGeneration(): raise CommandException('%s: a version-specific URL\n(%s)\ncannot be ' 'the destination for gsutil cp - abort.' % (cmd_name, dst_url)) if not dst_url.IsCloudUrl() and copy_helper_opts.dest_storage_class: raise CommandException('Cannot specify storage class for a non-cloud ' 'destination: %s' % dst_url) src_obj_metadata = None if copy_object_info.expanded_result: src_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, copy_object_info.expanded_result) if src_url.IsFileUrl() and preserve_posix: if not src_obj_metadata: src_obj_metadata = apitools_messages.Object() mode, _, _, _, uid, gid, _, atime, mtime, _ = os.stat( exp_src_url.object_name) mode = ConvertModeToBase8(mode) posix_attrs = POSIXAttributes(atime=atime, mtime=mtime, uid=uid, gid=gid, mode=mode) custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, custom_metadata, preserve_posix=preserve_posix) src_obj_metadata.metadata = custom_metadata if src_obj_metadata and dst_url.IsFileUrl(): posix_attrs = DeserializeFileAttributesFromObjectMetadata( src_obj_metadata, src_url.url_string) mode = posix_attrs.mode.permissions valid, err = ValidateFilePermissionAccess(src_url.url_string, uid=posix_attrs.uid, gid=posix_attrs.gid, mode=mode) if preserve_posix and not valid: logging.getLogger().critical(err) raise CommandException('This sync will orphan file(s), please fix their' ' permissions before trying again.') bytes_transferred = 0 try: if copy_helper_opts.use_manifest: self.manifest.Initialize(exp_src_url.url_string, dst_url.url_string) if (self.recursion_requested and copy_object_info.exp_dst_url.object_name and dst_url.IsFileUrl()): # exp_dst_url is the wildcard-expanded path passed by the user: # exp_dst_url => ~/dir # container => /usr/name/dir container = os.path.abspath(copy_object_info.exp_dst_url.object_name) # dst_url holds the complete path of the object's destination: # dst_url => /usr/name/dir/../file.txt # abspath => /usr/name/file.txt # # Taking the common path of this and container yields: /usr/name, # which does not start with container when the inclusion of '..' strings # results in a copy outside of the container. if not os.path.commonpath([ container, os.path.abspath(dst_url.object_name) ]).startswith(container): self.logger.warn( 'Skipping copy of source URL %s because it would be copied ' 'outside the expected destination directory: %s.' % (exp_src_url, container)) if copy_helper_opts.use_manifest: self.manifest.SetResult( exp_src_url.url_string, 0, 'skip', 'Would have copied outside the destination directory.') return _, bytes_transferred, result_url, md5 = copy_helper.PerformCopy( self.logger, exp_src_url, dst_url, gsutil_api, self, _CopyExceptionHandler, src_obj_metadata=src_obj_metadata, allow_splitting=True, headers=self.headers, manifest=self.manifest, gzip_encoded=self.gzip_encoded, gzip_exts=self.gzip_exts, preserve_posix=preserve_posix, use_stet=self.use_stet) if copy_helper_opts.use_manifest: if md5: self.manifest.Set(exp_src_url.url_string, 'md5', md5) self.manifest.SetResult(exp_src_url.url_string, bytes_transferred, 'OK') if copy_helper_opts.print_ver: # Some cases don't return a version-specific URL (e.g., if destination # is a file). self.logger.info('Created: %s', result_url) except ItemExistsError: message = 'Skipping existing item: %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except SkipUnsupportedObjectError as e: message = ('Skipping item %s with unsupported object type %s' % (exp_src_url.url_string, e.unsupported_type)) self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) except copy_helper.FileConcurrencySkipError as e: self.logger.warn( 'Skipping copy of source URL %s because destination URL ' '%s is already being copied by another gsutil process ' 'or thread (did you specify the same source URL twice?) ' % (src_url, dst_url)) except Exception as e: # pylint: disable=broad-except if (copy_helper_opts.no_clobber and copy_helper.IsNoClobberServerException(e)): message = 'Rejected (noclobber): %s' % dst_url self.logger.info(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message) elif self.continue_on_error: message = 'Error copying %s: %s' % (src_url, str(e)) self.op_failure_count += 1 self.logger.error(message) if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', RemoveCRLFFromString(message)) else: if copy_helper_opts.use_manifest: self.manifest.SetResult(exp_src_url.url_string, 0, 'error', str(e)) raise else: if copy_helper_opts.perform_mv: self.logger.info('Removing %s...', exp_src_url) if exp_src_url.IsCloudUrl(): gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, generation=exp_src_url.generation, provider=exp_src_url.scheme) else: os.unlink(exp_src_url.object_name) with self.stats_lock: # TODO: Remove stats_lock; we should be able to calculate bytes # transferred from StatusMessages posted by operations within PerformCopy. self.total_bytes_transferred += bytes_transferred def _ConstructNameExpansionIteratorDstTupleIterator(self, src_url_strs_iter, dst_url_strs): copy_helper_opts = copy_helper.GetCopyHelperOpts() for src_url_str, dst_url_str in zip(src_url_strs_iter, dst_url_strs): # Getting the destination information for each (sources, destination) # tuple. This assumes that the same destination is never provided in # multiple tuples, and doing so may result in an inconsistent behavior # especially when using the -m multi-threading option. # # Example for the inconsistent behavior, the following commands will # behave differently: # # gsutil cp -r dir1 dir2 gs://bucket/non-existent-dir # gsutil cp -r [ # (dir1, gs://bucket/non-existent-dir), # (dir2, gs://bucket/non-existent-dir) # ] # # When multiple threads execute on a non existing destination directory. # These threads might encounter different states of the destination # directory. The first thread to execute the command finds that the # destination directory does not exist, it will create the destination # directory and copies the files inside the source directories to the # destination directory. The following threads find that the destination # directory already exists and copy the source directories in the # destination directory. In another scenario, all the threads might find # that the destination directory does not exist and copy the source # directories to the destination directory. exp_dst_url, have_existing_dst_container = ( copy_helper.ExpandUrlToSingleBlr(dst_url_str, self.gsutil_api, self.project_id, logger=self.logger)) name_expansion_iterator_dst_tuple = NameExpansionIteratorDestinationTuple( NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, src_url_str, self.recursion_requested or copy_helper_opts.perform_mv, project_id=self.project_id, all_versions=self.all_versions, ignore_symlinks=self.exclude_symlinks, continue_on_error=(self.continue_on_error or self.parallel_operations), bucket_listing_fields=GetSourceFieldsNeededForCopy( exp_dst_url.IsCloudUrl(), copy_helper_opts.skip_unsupported_objects, copy_helper_opts.preserve_acl, preserve_posix=self.preserve_posix_attrs, delete_source=copy_helper_opts.perform_mv, file_size_will_change=self.use_stet)), DestinationInfo(exp_dst_url, have_existing_dst_container)) self.has_file_dst = self.has_file_dst or exp_dst_url.IsFileUrl() self.has_cloud_dst = self.has_cloud_dst or exp_dst_url.IsCloudUrl() self.provider_types.add(exp_dst_url.scheme) self.combined_src_urls = itertools.chain(self.combined_src_urls, src_url_str) yield name_expansion_iterator_dst_tuple # Command entry point. def RunCommand(self): copy_helper_opts = self._ParseOpts() self.total_bytes_transferred = 0 dst_url = StorageUrlFromString(self.args[-1]) if dst_url.IsFileUrl() and (dst_url.object_name == '-' or dst_url.IsFifo()): if self.preserve_posix_attrs: raise CommandException('Cannot preserve POSIX attributes with a ' 'stream or a named pipe.') cat_out_fd = (GetStreamFromFileUrl(dst_url, mode='wb') if dst_url.IsFifo() else None) return cat_helper.CatHelper(self).CatUrlStrings(self.args[:-1], cat_out_fd=cat_out_fd) if copy_helper_opts.read_args_from_stdin: if len(self.args) != 1: raise CommandException('Source URLs cannot be specified with -I option') # Use StdinIteratorCls instead of StdinIterator here to avoid Python 3 # generator pickling errors when multiprocessing a command. src_url_strs = [StdinIteratorCls()] else: if len(self.args) < 2: raise CommandException('Wrong number of arguments for "cp" command.') src_url_strs = [self.args[:-1]] dst_url_strs = [self.args[-1]] self.combined_src_urls = [] self.has_file_dst = False self.has_cloud_dst = False self.provider_types = set() # Because cp may have multiple source URLs and multiple destinations, we # wrap the name expansion iterator in order to collect analytics. name_expansion_iterator = CopyObjectsIterator( self._ConstructNameExpansionIteratorDstTupleIterator( src_url_strs, dst_url_strs), copy_helper_opts.daisy_chain, ) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=None, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, process_count * thread_count) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not copy_helper_opts.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.combined_src_urls, self.recursion_requested or copy_helper_opts.perform_mv, all_versions=self.all_versions, project_id=self.project_id, ignore_symlinks=self.exclude_symlinks, file_size_will_change=self.use_stet) # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() # Tracks if any copies failed. self.op_failure_count = 0 # Start the clock. start_time = time.time() # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') # Perform copy requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_CopyFuncWrapper, name_expansion_iterator, _CopyExceptionHandler, shared_attrs, fail_on_error=(not self.continue_on_error), seek_ahead_iterator=seek_ahead_iterator) self.logger.debug('total_bytes_transferred: %d', self.total_bytes_transferred) end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( has_file_dst=self.has_file_dst, has_cloud_dst=self.has_cloud_dst, avg_throughput=self.total_bytes_per_second, total_bytes_transferred=self.total_bytes_transferred, total_elapsed_time=self.total_elapsed_time, uses_fan=self.parallel_operations, is_daisy_chain=copy_helper_opts.daisy_chain, provider_types=list(self.provider_types)) if self.debug >= DEBUGLEVEL_DUMP_REQUESTS: # Note that this only counts the actual GET and PUT bytes for the copy # - not any transfers for doing wildcard expansion, the initial # HEAD/GET request performed to get the object metadata, etc. if self.total_bytes_transferred != 0: self.logger.info( 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)', self.total_bytes_transferred, self.total_elapsed_time, MakeHumanReadable(self.total_bytes_per_second)) if self.op_failure_count: plural_str = 's' if self.op_failure_count > 1 else '' raise CommandException('{count} file{pl}/object{pl} could ' 'not be transferred.'.format( count=self.op_failure_count, pl=plural_str)) return 0 def _ParseOpts(self): # TODO: Arrange variables initialized here in alphabetical order. perform_mv = False # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False no_clobber = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False daisy_chain = False read_args_from_stdin = False print_ver = False use_manifest = False preserve_acl = False self.preserve_posix_attrs = False canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None self.all_versions = False self.skip_unsupported_objects = False # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. The gzip_local flag marks if the files should be compressed # before uploading. Files compressed prior to uploaded are stored # compressed, while files compressed during the upload are stored # uncompressed. These flags cannot be mixed. gzip_encoded = False gzip_local = False gzip_arg_exts = None gzip_arg_all = None test_callback_file = None dest_storage_class = None self.use_stet = False # self.recursion_requested initialized in command.py (so can be checked # in parent class for all commands). self.manifest = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-A': self.all_versions = True if o == '-c': self.continue_on_error = True elif o == '-D': daisy_chain = True elif o == '-e': self.exclude_symlinks = True elif o == '--testcallbackfile': # File path of a pickled class that implements ProgressCallback.call. # Used for testing transfer interruptions and resumes. test_callback_file = a elif o == '-I': read_args_from_stdin = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-L': use_manifest = True self.manifest = Manifest(a) elif o == '-M': # Note that we signal to the cp command to perform a move (copy # followed by remove) and use directory-move naming rules by passing # the undocumented (for internal use) -M option when running the cp # command from mv.py. perform_mv = True elif o == '-n': no_clobber = True elif o == '-p': preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-s': dest_storage_class = NormalizeStorageClass(a) elif o == '-U': self.skip_unsupported_objects = True elif o == '-v': print_ver = True elif o == '-z': gzip_local = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-Z': gzip_local = True gzip_arg_all = GZIP_ALL_FILES elif o == '--stet': self.use_stet = True if preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if self.all_versions and self.parallel_operations: raise CommandException( 'The gsutil -m option is not supported with the cp -A flag, to ' 'ensure that object version ordering is preserved. Please re-run ' 'the command without the -m option.') if gzip_encoded and gzip_local: raise CommandException( 'Specifying both the -j/-J and -z/-Z options together is invalid.') if gzip_arg_exts and gzip_arg_all: if gzip_encoded: raise CommandException( 'Specifying both the -j and -J options together is invalid.') else: raise CommandException( 'Specifying both the -z and -Z options together is invalid.') self.gzip_exts = gzip_arg_exts or gzip_arg_all self.gzip_encoded = gzip_encoded return CreateCopyHelperOpts( perform_mv=perform_mv, no_clobber=no_clobber, daisy_chain=daisy_chain, read_args_from_stdin=read_args_from_stdin, print_ver=print_ver, use_manifest=use_manifest, preserve_acl=preserve_acl, canned_acl=canned_acl, skip_unsupported_objects=self.skip_unsupported_objects, test_callback_file=test_callback_file, dest_storage_class=dest_storage_class)	57.54 KB	2025-06-25 08:19
📄 defacl.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of default object acl command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os from gslib import gcs_json_api from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import Preconditions from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import SetAclExceptionHandler from gslib.command import SetAclFuncWrapper from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import acl_helper from gslib.utils.constants import NO_MAX from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL _SET_SYNOPSIS = """ gsutil defacl set (<file-path>\|<predefined-acl>) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defacl get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil defacl ch [-f] -u\|-g\|-d\|-p <grant>... gs://<bucket_name>... """ _SET_DESCRIPTION = """ <B>SET</B> The ``defacl set`` command sets default object ACLs for the specified buckets. If you specify a default object ACL for a certain bucket, Cloud Storage applies the default object ACL to all new objects uploaded to that bucket, unless an ACL for that object is separately specified during upload. Similar to the ``acl set`` command, the ``defacl set`` command specifies either a predefined ACL or the path to a file that contains ACL text. See "gsutil help acl" for examples of editing and setting ACLs via the acl command. See `Predefined ACLs <https://cloud.google.com/storage/docs/access-control/lists#predefined-acl>`_ for a list of predefined ACLs. Setting a default object ACL on a bucket provides a convenient way to ensure newly uploaded objects have a specific ACL. If you don't set the bucket's default object ACL, it will default to project-private. If you then upload objects that need a different ACL, you will need to perform a separate ACL update operation for each object. Depending on how many objects require updates, this could be very time-consuming. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default ACL text for a bucket, which you can save and edit for use with the "defacl set" command. """ _CH_DESCRIPTION = """ <B>CH</B> The "defacl ch" (or "defacl change") command updates the default object access control list for a bucket. The syntax is shared with the "acl ch" command, so see the "CH" section of "gsutil help acl" for the full help description. <B>CH EXAMPLES</B> Grant anyone on the internet READ access by default to any object created in the bucket example-bucket: gsutil defacl ch -u AllUsers:R gs://example-bucket NOTE: By default, publicly readable objects are served with a Cache-Control header allowing such objects to be cached for 3600 seconds. If you need to ensure that updates become visible immediately, you should set a Cache-Control header of "Cache-Control:private, max-age=0, no-transform" on such objects. For help doing this, see "gsutil help setmeta". Add the user john.doe@example.com to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -u john.doe@example.com:READ gs://example-bucket Add the group admins@example.com to the default object ACL on bucket example-bucket with OWNER access: gsutil defacl ch -g admins@example.com:O gs://example-bucket Remove the group admins@example.com from the default object ACL on bucket example-bucket: gsutil defacl ch -d admins@example.com gs://example-bucket Add the owners of project example-project-123 to the default object ACL on bucket example-bucket with READ access: gsutil defacl ch -p owners-example-project-123:R gs://example-bucket NOTE: You can replace 'owners' with 'viewers' or 'editors' to grant access to a project's viewers/editors respectively. <B>CH OPTIONS</B> The "ch" sub-command has the following options -d Remove all roles associated with the matching entity. -f Normally gsutil stops at the first error. The -f option causes it to continue when it encounters errors. With this option the gsutil exit status will be 0 even if some ACLs couldn't be changed. -g Add or modify a group entity's role. -p Add or modify a project viewers/editors/owners role. -u Add or modify a user entity's role. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ The defacl command has three sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION + _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class DefAclCommand(Command): """Implementation of gsutil defacl command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defacl', command_name_aliases=['setdefacl', 'getdefacl', 'chdefacl'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='fg:u:d:p:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeFileURLOrCannedACLArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defacl', help_name_aliases=['default acl', 'setdefacl', 'getdefacl', 'chdefacl'], help_type='command_help', help_one_line_summary='Get, set, or change default ACL on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def get_gcloud_storage_args(self): sub_command = self.args.pop(0) if sub_command == 'get': gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=multi(defaultObjectAcl:format=json)', '--raw' ], flag_map={}, ) elif sub_command == 'set': acl_file_or_predefined_acl = self.args.pop(0) if (os.path.isfile(acl_file_or_predefined_acl)): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-object-acl-file=' + acl_file_or_predefined_acl ], flag_map={}, ) else: if acl_file_or_predefined_acl in ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION): predefined_acl = ( gcs_json_api.FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[ acl_file_or_predefined_acl]) else: predefined_acl = acl_file_or_predefined_acl gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--predefined-default-object-acl=' + predefined_acl ], flag_map={}, ) elif sub_command == 'ch': self.ParseSubOpts() self.sub_opts = acl_helper.translate_sub_opts_for_shim(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-g': GcloudStorageFlag('--add-default-object-acl-grant'), '-p': GcloudStorageFlag('--add-default-object-acl-grant'), '-u': GcloudStorageFlag('--add-default-object-acl-grant'), '-d': GcloudStorageFlag('--remove-default-object-acl-grant'), '-f': GcloudStorageFlag('--continue-on-error'), }) return super().get_gcloud_storage_args(gcloud_storage_map) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if (self.args[0].lower() == 'set' or self.command_alias_used == 'setdefacl'): return 1 else: return 0 def _SetDefAcl(self): if not StorageUrlFromString(self.args[-1]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) try: self.SetAclCommandHelper(SetAclFuncWrapper, SetAclExceptionHandler) except AccessDeniedException: self._WarnServiceAccounts() raise def _GetDefAcl(self): if not StorageUrlFromString(self.args[0]).IsBucket(): raise CommandException('URL must name a bucket for the %s command' % self.command_name) self.GetAndPrintAcl(self.args[0]) def _ChDefAcl(self): """Parses options and changes default object ACLs on specified buckets.""" self.parse_versions = True self.changes = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-g': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.GROUP)) if o == '-u': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.USER)) if o == '-p': self.changes.append( acl_helper.AclChange(a, scope_type=acl_helper.ChangeType.PROJECT)) if o == '-d': self.changes.append(acl_helper.AclDel(a)) if not self.changes: raise CommandException('Please specify at least one access change ' 'with the -g, -u, or -d flags') if (not UrlsAreForSingleProvider(self.args) or StorageUrlFromString(self.args[0]).scheme != 'gs'): raise CommandException( 'The "{0}" command can only be used with gs:// URLs'.format( self.command_name)) bucket_urls = set() for url_arg in self.args: for result in self.WildcardIterator(url_arg): if not result.storage_url.IsBucket(): raise CommandException( 'The defacl ch command can only be applied to buckets.') bucket_urls.add(result.storage_url) for storage_url in bucket_urls: self.ApplyAclChanges(storage_url) @Retry(ServiceException, tries=3, timeout_secs=1) def ApplyAclChanges(self, url): """Applies the changes in self.changes to the provided URL.""" bucket = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['defaultObjectAcl', 'metageneration']) # Default object ACLs can be blank if the ACL was set to private, or # if the user doesn't have permission. We warn about this with defacl get, # so just try the modification here and if the user doesn't have # permission they'll get an AccessDeniedException. current_acl = bucket.defaultObjectAcl if self._ApplyAclChangesAndReturnChangeCount(url, current_acl) == 0: self.logger.info('No changes to %s', url) return if not current_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. current_acl.append(PRIVATE_DEFAULT_OBJ_ACL) try: preconditions = Preconditions(meta_gen_match=bucket.metageneration) bucket_metadata = apitools_messages.Bucket(defaultObjectAcl=current_acl) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) self.logger.info('Updated default ACL on %s', url) except BadRequestException as e: # Don't retry on bad requests, e.g. invalid email address. raise CommandException('Received bad request from server: %s' % str(e)) except AccessDeniedException: self._WarnServiceAccounts() raise CommandException('Failed to set acl for %s. Please ensure you have ' 'OWNER-role access to this resource.' % url) def _ApplyAclChangesAndReturnChangeCount(self, storage_url, defacl_message): modification_count = 0 for change in self.changes: modification_count += change.Execute(storage_url, defacl_message, 'defacl', self.logger) return modification_count def RunCommand(self): """Command entry point for the defacl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) self.def_acl = True self.continue_on_error = False if action_subcommand == 'get': func = self._GetDefAcl elif action_subcommand == 'set': func = self._SetDefAcl elif action_subcommand in ('ch', 'change'): func = self._ChDefAcl else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help defacl".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) func() return 0	14.53 KB	2025-06-25 08:19
📄 defstorageclass.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Impl. of default bucket storage class command for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil defstorageclass set <storage-class> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil defstorageclass get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "defstorageclass set" command sets the default `storage class <https://cloud.google.com/storage/docs/storage-classes>`_ for the specified bucket(s). If you specify a default storage class for a certain bucket, Google Cloud Storage applies the default storage class to all new objects uploaded to that bucket, except when the storage class is overridden by individual upload requests. Setting a default storage class on a bucket provides a convenient way to ensure newly uploaded objects have a specific storage class. If you don't set the bucket's default storage class, it will default to Standard. """ _GET_DESCRIPTION = """ <B>GET</B> Gets the default storage class for a bucket. """ _DESCRIPTION = """ The defstorageclass command has two sub-commands: """ + '\n'.join([_SET_DESCRIPTION + _GET_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) # The url_string for buckets ends with a slash. # Substitute the last slash with a colon. _GCLOUD_FORMAT_STRING = ('--format=value[separator=\": \"](name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' 'storageClass)') SHIM_GET_COMMAND_MAP = GcloudStorageMap( # Using a list because a string gets splitted up on space and the # format string below has a space. gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ) SHIM_SET_COMMAND_MAP = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--default-storage-class'], flag_map={}, ) class DefStorageClassCommand(Command): """Implementation of gsutil defstorageclass command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'defstorageclass', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ # FreeTextArgument allows for using storage class abbreviations. CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='defstorageclass', help_name_aliases=['defaultstorageclass'], help_type='command_help', help_one_line_summary='Get or set the default storage class on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': SHIM_GET_COMMAND_MAP, 'set': SHIM_SET_COMMAND_MAP, }, flag_map={}, ) def _CheckIsGsUrl(self, url_str): if not url_str.startswith('gs://'): raise CommandException( '"%s" does not support the URL "%s". Did you mean to use a gs:// ' 'URL?' % (self.command_name, url_str)) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetDefStorageClass(self): """Sets the default storage class for a bucket.""" # At this point, "set" has been popped off the front of self.args. normalized_storage_class = NormalizeStorageClass(self.args[0]) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: some_matched = True bucket_metadata = apitools_messages.Bucket() self.logger.info('Setting default storage class to "%s" for bucket %s' % (normalized_storage_class, blr.url_string.rstrip('/'))) bucket_metadata.storageClass = normalized_storage_class self.gsutil_api.PatchBucket(blr.storage_url.bucket_name, bucket_metadata, provider=blr.storage_url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetDefStorageClass(self): """Gets the default storage class for a bucket.""" # At this point, "get" has been popped off the front of self.args. url_args = self.args some_matched = False for url_str in url_args: self._CheckIsGsUrl(url_str) bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['storageClass']) for blr in bucket_iter: some_matched = True print('%s: %s' % (blr.url_string.rstrip('/'), blr.root_object.storageClass)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the defstorageclass command.""" action_subcommand = self.args.pop(0) subcommand_args = [action_subcommand] if action_subcommand == 'get': func = self._GetDefStorageClass elif action_subcommand == 'set': func = self._SetDefStorageClass normalized_storage_class = NormalizeStorageClass(self.args[0]) subcommand_args.append(normalized_storage_class) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) metrics.LogCommandParams(subcommands=subcommand_args) func() return 0	7.73 KB	2025-06-25 08:19
📄 du.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like du command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import locale import sys import six from gslib.bucket_listing_ref import BucketListingObject from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import ls_helper from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import print_to_fd from gslib.utils.unit_util import MakeHumanReadable from gslib.utils import text_util _SYNOPSIS = """ gsutil du url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The du command displays the amount of space in bytes used up by the objects in a bucket, subdirectory, or project. The syntax emulates the Linux ``du -b`` command, which reports the disk usage of files and subdirectories. For example, the following command reports the total space used by all objects and subdirectories under gs://your-bucket/dir: gsutil du -s -a gs://your-bucket/dir <B>OPTIONS</B> -0 Ends each output line with a 0 byte rather than a newline. You can use this to make the output machine-readable. -a Includes both live and noncurrent object versions. Also prints the generation and metageneration number for each listed object. If this flag is not specified, only live object versions are included. -c Includes a total size at the end of the output. -e Exclude a pattern from the report. Example: -e ".o" excludes any object that ends in ".o". Can be specified multiple times. -h Prints object sizes in human-readable format. For example, ``1 KiB``, ``234 MiB``, or ``2GiB``. -s Displays only the total size for each argument, omitting the list of individual objects. -X Similar to ``-e``, but excludes patterns from the given file. The patterns to exclude should be listed one per line. <B>EXAMPLES</B> To list the size of each object in a bucket: gsutil du gs://bucketname To list the size of each object in the ``prefix`` subdirectory: gsutil du gs://bucketname/prefix/ To include the total number of bytes in human-readable form: gsutil du -ch gs://bucketname To see only the summary of the total number of (live) bytes in two given buckets: gsutil du -s gs://bucket1 gs://bucket2 To list the size of each object in a bucket with `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_ enabled, including noncurrent objects: gsutil du -a gs://bucketname To list the size of each object in a bucket, except objects that end in ".bak", with each object printed ending in a null byte: gsutil du -e ".bak" -0 gs://bucketname To list the size of each bucket in a project and the total size of the project: gsutil -o GSUtil:default_project_id=project-name du -shc """) class DuCommand(Command): """Implementation of gsutil du command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'du', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='0ace:hsX:', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='du', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object size usage', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'du'], flag_map={ '-0': GcloudStorageFlag('--zero-terminator'), '-a': GcloudStorageFlag('--all-versions'), '-c': GcloudStorageFlag('--total'), '-e': GcloudStorageFlag('--exclude-name-pattern'), '-h': GcloudStorageFlag('--readable-sizes'), '-s': GcloudStorageFlag('--summarize'), '-X': GcloudStorageFlag('--exclude-name-pattern-file'), }, ) def _PrintSummaryLine(self, num_bytes, name): size_string = (MakeHumanReadable(num_bytes) if self.human_readable else six.text_type(num_bytes)) text_util.print_to_fd('{size:<11} {name}'.format( size=size_string, name=six.ensure_text(name)), end=self.line_ending) def _PrintInfoAboutBucketListingRef(self, bucket_listing_ref): """Print listing info for given bucket_listing_ref. Args: bucket_listing_ref: BucketListing being listed. Returns: Tuple (number of objects, object size) Raises: Exception: if calling bug encountered. """ obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 if not self.summary_only: url_detail = '{size:<11} {url}{ending}'.format( size=size_string, url=six.ensure_text(url_str), ending=six.ensure_text(self.line_ending)) print_to_fd(url_detail, file=sys.stdout, end='') return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the du command.""" self.line_ending = '\n' self.all_versions = False self.produce_total = False self.human_readable = False self.summary_only = False self.exclude_patterns = [] if self.sub_opts: for o, a in self.sub_opts: if o == '-0': self.line_ending = '\0' elif o == '-a': self.all_versions = True elif o == '-c': self.produce_total = True elif o == '-e': self.exclude_patterns.append(a) elif o == '-h': self.human_readable = True elif o == '-s': self.summary_only = True elif o == '-X': if a == '-': f = sys.stdin f_close = False else: f = open(a, 'r') if six.PY2 else open(a, 'r', encoding=UTF8) f_close = True self.exclude_patterns = [six.ensure_text(line.strip()) for line in f] if f_close: f.close() if not self.args: # Default to listing all gs buckets. self.args = ['gs://'] total_bytes = 0 got_nomatch_errors = False def _PrintObjectLong(blr): return self._PrintInfoAboutBucketListingRef(blr) def _PrintNothing(unused_blr=None): pass def _PrintDirectory(num_bytes, blr): if not self.summary_only: self._PrintSummaryLine(num_bytes, blr.url_string.encode(UTF8)) for url_arg in self.args: top_level_storage_url = StorageUrlFromString(url_arg) if top_level_storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_listing_fields = ['size'] listing_helper = ls_helper.LsHelper( self.WildcardIterator, self.logger, print_object_func=_PrintObjectLong, print_dir_func=_PrintNothing, print_dir_header_func=_PrintNothing, print_dir_summary_func=_PrintDirectory, print_newline_func=_PrintNothing, all_versions=self.all_versions, should_recurse=True, exclude_patterns=self.exclude_patterns, fields=bucket_listing_fields) # LsHelper expands to objects and prefixes, so perform a top-level # expansion first. if top_level_storage_url.IsProvider(): # Provider URL: use bucket wildcard to iterate over all buckets. top_level_iter = self.WildcardIterator( '%s://' % top_level_storage_url.scheme).IterBuckets(bucket_fields=['id']) elif top_level_storage_url.IsBucket(): top_level_iter = self.WildcardIterator( '%s://%s' % (top_level_storage_url.scheme, top_level_storage_url.bucket_name)).IterBuckets( bucket_fields=['id']) else: top_level_iter = [BucketListingObject(top_level_storage_url)] for blr in top_level_iter: storage_url = blr.storage_url if storage_url.IsBucket() and self.summary_only: storage_url = StorageUrlFromString( storage_url.CreatePrefixUrl(wildcard_suffix='**')) _, exp_objs, exp_bytes = listing_helper.ExpandUrlAndPrint(storage_url) if (storage_url.IsObject() and exp_objs == 0 and ContainsWildcard(url_arg) and not self.exclude_patterns): got_nomatch_errors = True total_bytes += exp_bytes if self.summary_only: self._PrintSummaryLine(exp_bytes, blr.url_string.rstrip('/').encode(UTF8)) if self.produce_total: self._PrintSummaryLine(total_bytes, 'total') if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') return 0	10.48 KB	2025-06-25 08:19
📄 hash.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of hash command for calculating hashes of local files.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import time import crcmod import six from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.progress_callback import FileProgressCallbackHandler from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.storage_url import StorageUrlFromString from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import hashing_helper from gslib.utils import parallelism_framework_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil hash [-c] [-h] [-m] filename... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Calculate hashes on local files, which can be used to compare with ``gsutil ls -L`` output. If a specific hash option is not provided, this command calculates all gsutil-supported hashes for the files. Note that gsutil automatically performs hash validation when uploading or downloading files, so this command is only needed if you want to write a script that separately checks the hash. If you calculate a CRC32c hash for files without a precompiled crcmod installation, hashing will be very slow. See "gsutil help crcmod" for details. <B>OPTIONS</B> -c Calculate a CRC32c hash for the specified files. -h Output hashes in hex format. By default, gsutil uses base64. -m Calculate a MD5 hash for the specified files. """) _GCLOUD_FORMAT_STRING = ( '--format=' + 'value[separator="",terminator=""](' + 'digest_format.sub("' + shim_util.get_format_flag_caret() + '", "Hashes ["),' + 'url.sub("' + shim_util.get_format_flag_caret() + '", "] for ").sub("$", ":^\\^n"),' + 'md5_hash.yesno(yes="\tHash (md5):\t\t", no=""),' + 'md5_hash.yesno(no=""),' + 'md5_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '"),' + 'crc32c_hash.yesno(yes="\tHash (crc32c):\t\t", no=""),' + 'crc32c_hash.yesno(no=""),' + 'crc32c_hash.yesno(yes="NEWLINE", no="")' + '.sub("NEWLINE", "' + shim_util.get_format_flag_newline() + '")' + ')') class HashCommand(Command): """Implementation of gsutil hash command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'hash', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='chm', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreFileURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='hash', help_name_aliases=['checksum'], help_type='command_help', help_one_line_summary='Calculate file hashes', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): gcloud_storage_map = GcloudStorageMap( gcloud_command=[ 'storage', 'hash', _GCLOUD_FORMAT_STRING, ], flag_map={ '-h': GcloudStorageFlag('--hex'), '-c': None, '-m': None, }, ) args_set = set(self.args + [flag for flag, _ in self.sub_opts]) if '-c' in args_set and '-m' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-md5'] elif '-m' in args_set and '-c' not in args_set: gcloud_storage_map.gcloud_command += ['--skip-crc32c'] return super().get_gcloud_storage_args(gcloud_storage_map) @classmethod def _ParseOpts(cls, sub_opts, logger): """Returns behavior variables based on input options. Args: sub_opts: getopt sub-arguments for the command. logger: logging.Logger for the command. Returns: Tuple of calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum. calc_md5: Boolean, if True, command should calculate an MD5 hash. format_func: Function used for formatting the hash in the desired format. cloud_format_func: Function used for formatting the hash in the desired format. output_format: String describing the hash output format. """ calc_crc32c = False calc_md5 = False format_func = ( lambda digest: hashing_helper.Base64EncodeHash(digest.hexdigest())) cloud_format_func = lambda digest: digest found_hash_option = False output_format = 'base64' if sub_opts: for o, unused_a in sub_opts: if o == '-c': calc_crc32c = True found_hash_option = True elif o == '-h': output_format = 'hex' format_func = lambda digest: digest.hexdigest() cloud_format_func = lambda digest: ( hashing_helper.Base64ToHexHash(digest).decode('ascii') ) # yapf: disable elif o == '-m': calc_md5 = True found_hash_option = True if not found_hash_option: calc_crc32c = True calc_md5 = True if calc_crc32c and not boto_util.UsingCrcmodExtension(): logger.warn(hashing_helper.SLOW_CRCMOD_WARNING) return calc_crc32c, calc_md5, format_func, cloud_format_func, output_format def _GetHashClassesFromArgs(self, calc_crc32c, calc_md5): """Constructs the dictionary of hashes to compute based on the arguments. Args: calc_crc32c: If True, CRC32c should be included. calc_md5: If True, MD5 should be included. Returns: Dictionary of {string: hash digester}, where string the name of the digester algorithm. """ hash_dict = {} if calc_crc32c: hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') if calc_md5: hash_dict['md5'] = hashing_helper.GetMd5() return hash_dict def RunCommand(self): """Command entry point for the hash command.""" (calc_crc32c, calc_md5, format_func, cloud_format_func, output_format) = (self._ParseOpts(self.sub_opts, self.logger)) matched_one = False for url_str in self.args: for file_ref in self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=[ 'crc32c', 'customerEncryption', 'md5Hash', 'size', ]): matched_one = True url = StorageUrlFromString(url_str) file_name = file_ref.storage_url.object_name if StorageUrlFromString(url_str).IsFileUrl(): file_size = os.path.getsize(file_name) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=False, message_type=FileMessage.FILE_HASH)) callback_processor = ProgressCallbackWithTimeout( file_size, FileProgressCallbackHandler(self.gsutil_api.status_queue, src_url=StorageUrlFromString(url_str), operation_name='Hashing').call) hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5) with open(file_name, 'rb') as fp: hashing_helper.CalculateHashesFromContents( fp, hash_dict, callback_processor=callback_processor) self.gsutil_api.status_queue.put( FileMessage(url, None, time.time(), size=file_size, finished=True, message_type=FileMessage.FILE_HASH)) else: hash_dict = {} obj_metadata = file_ref.root_object file_size = obj_metadata.size md5_present = obj_metadata.md5Hash is not None crc32c_present = obj_metadata.crc32c is not None if not md5_present and not crc32c_present: logging.getLogger().warn('No hashes present for %s', url_str) continue if md5_present: hash_dict['md5'] = obj_metadata.md5Hash if crc32c_present: hash_dict['crc32c'] = obj_metadata.crc32c print('Hashes [%s] for %s:' % (output_format, file_name)) for name, digest in six.iteritems(hash_dict): print('\tHash (%s):\t\t%s' % (name, (format_func(digest) if url.IsFileUrl() else cloud_format_func(digest)))) if not matched_one: raise CommandException('No files matched') _PutToQueueWithTimeout(self.gsutil_api.status_queue, FinalMessage(time.time())) return 0	9.82 KB	2025-06-25 08:19
📄 help.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil help command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import os import pkgutil import re import six from subprocess import PIPE from subprocess import Popen import gslib.addlhelp from gslib.command import Command from gslib.command import OLD_ALIAS_MAP import gslib.commands from gslib.exception import CommandException from gslib.help_provider import HelpProvider from gslib.help_provider import MAX_HELP_NAME_LEN from gslib.utils import constants from gslib.utils.system_util import IS_WINDOWS from gslib.utils.system_util import IsRunningInteractively from gslib.utils.system_util import GetTermLines from gslib.utils import text_util _SYNOPSIS = """ gsutil help [command or topic] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The following command provides a summary of gsutil commands and additional topics on which help is available: gsutil help The following command provides help about the specified command or topic: gsutil help [command or topic] The following command provides help about the specified sub-command: gsutil help [command] [sub-command] For example, running the following provides help about the "set" sub-command of the "acl" command: gsutil help acl set If you set the PAGER environment variable to the path to a pager program (such as /bin/less on Linux), long help sections are piped through the specified pager. """) top_level_usage_string = ( 'Usage: gsutil [-D] [-DD] [-h header]... [-i service_account] ' '[-m] [-o section:flag=value]... [-q] [-u user_project] [command [opts...] args...]' ) class HelpCommand(Command): """Implementation of gsutil help command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'help', command_name_aliases=['?', 'man'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=2, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='help', help_name_aliases=['?'], help_type='command_help', help_one_line_summary='Get help about commands and topics', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the help command.""" (help_type_map, help_name_map) = self._LoadHelpMaps() output = [] if not self.args: output.append('%s\nAvailable commands:\n' % top_level_usage_string) format_str = ' %-' + str(MAX_HELP_NAME_LEN) + 's%s\n' for help_prov in sorted(help_type_map['command_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nAdditional help topics:\n') for help_prov in sorted(help_type_map['additional_help'], key=lambda hp: hp.help_spec.help_name): output.append(format_str % (help_prov.help_spec.help_name, help_prov.help_spec.help_one_line_summary)) output.append('\nUse gsutil help <command or topic> for detailed help.') else: invalid_subcommand = False arg = self.args[0] if arg not in help_name_map: output.append('No help available for "%s"' % arg) else: help_prov = help_name_map[arg] help_name = None if len(self.args) > 1: # We also have a subcommand argument. subcommand_map = help_prov.help_spec.subcommand_help_text if subcommand_map and self.args[1] in subcommand_map: help_name = arg + ' ' + self.args[1] help_text = subcommand_map[self.args[1]] else: invalid_subcommand = True if not subcommand_map: output.append( ('The "%s" command has no subcommands. You can ask for the ' 'full help by running:\n\n\tgsutil help %s\n') % (arg, arg)) else: subcommand_examples = [] for subcommand in subcommand_map: subcommand_examples.append('\tgsutil help %s %s' % (arg, subcommand)) output.append( ('Subcommand "%s" does not exist for command "%s".\n' 'You can either ask for the full help about the command by ' 'running:\n\n\tgsutil help %s\n\n' 'Or you can ask for help about one of the subcommands:\n\n%s' ) % (self.args[1], arg, arg, '\n'.join(subcommand_examples))) if not invalid_subcommand: if not help_name: # No subcommand or invalid subcommand. help_name = help_prov.help_spec.help_name help_text = help_prov.help_spec.help_text output.append('<B>NAME</B>\n') output.append(' %s - %s\n' % (help_name, help_prov.help_spec.help_one_line_summary)) output.append('\n\n') output.append(help_text.strip('\n')) new_alias = OLD_ALIAS_MAP.get(arg, [None])[0] if new_alias: deprecation_warning = """ The "%s" alias is deprecated, and will eventually be removed completely. Please use the "%s" command instead.""" % (arg, new_alias) output.append('\n\n\n<B>DEPRECATION WARNING</B>\n') output.append(deprecation_warning) self._OutputHelp(''.join(output)) return 0 def _OutputHelp(self, help_str): """Outputs simply formatted string. This function paginates if the string is too long, PAGER is defined, and the output is a tty. Args: help_str: String to format. """ # Remove <B> and </B> tags and replace them with ANSI control codes if # writing to a compatible tty. if IS_WINDOWS or not IsRunningInteractively(): help_str = re.sub('<B>', '', help_str) help_str = re.sub('</B>', '', help_str) text_util.print_to_fd(help_str) return help_str = re.sub('<B>', '\033[1m', help_str) help_str = re.sub('</B>', '\033[0;0m', help_str) num_lines = len(help_str.split('\n')) if 'PAGER' in os.environ and num_lines >= GetTermLines(): # Use -r option for less to make bolding work right. pager = os.environ['PAGER'].split(' ') if pager[0].endswith('less'): pager.append('-r') try: if six.PY2: input_for_pager = help_str.encode(constants.UTF8) else: input_for_pager = help_str Popen(pager, stdin=PIPE, universal_newlines=True).communicate(input=input_for_pager) except OSError as e: raise CommandException('Unable to open pager (%s): %s' % (' '.join(pager), e)) else: text_util.print_to_fd(help_str) def _LoadHelpMaps(self): """Returns tuple of help type and help name. help type is a dict with key: help type value: list of HelpProviders help name is a dict with key: help command name or alias value: HelpProvider Returns: (help type, help name) """ # Import all gslib.commands submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): __import__('gslib.commands.%s' % module_name) # Import all gslib.addlhelp submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.addlhelp.__path__): __import__('gslib.addlhelp.%s' % module_name) help_type_map = {} help_name_map = {} for s in gslib.help_provider.ALL_HELP_TYPES: help_type_map[s] = [] # Only include HelpProvider subclasses in the dict. for help_prov in itertools.chain(HelpProvider.__subclasses__(), Command.__subclasses__()): if help_prov is Command: # Skip the Command base class itself; we just want its subclasses, # where the help command text lives (in addition to non-Command # HelpProviders, like naming.py). continue gslib.help_provider.SanityCheck(help_prov, help_name_map) help_name_map[help_prov.help_spec.help_name] = help_prov for help_name_aliases in help_prov.help_spec.help_name_aliases: help_name_map[help_name_aliases] = help_prov help_type_map[help_prov.help_spec.help_type].append(help_prov) return (help_type_map, help_name_map)	9.18 KB	2025-06-25 08:19
📄 hmac.py # -- coding: utf-8 -- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Implementation of HMAC key management command for GCS. NOTE: Any modification to this file or corresponding HMAC logic should be submitted in its own PR and release to avoid concurrency issues in testing. """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.project_id import PopulateProjectId from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import shim_util _CREATE_SYNOPSIS = """ gsutil hmac create [-p <project>] <service_account_email> """ _DELETE_SYNOPSIS = """ gsutil hmac delete [-p <project>] <access_id> """ _GET_SYNOPSIS = """ gsutil hmac get [-p <project>] <access_id> """ _LIST_SYNOPSIS = """ gsutil hmac list [-a] [-l] [-p <project>] [-u <service_account_email>] """ _UPDATE_SYNOPSIS = """ gsutil hmac update -s (ACTIVE\|INACTIVE) [-e <etag>] [-p <project>] <access_id> """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The ``hmac create`` command creates an HMAC key for the specified service account: gsutil hmac create test.service.account@test_project.iam.gserviceaccount.com The secret key material is only available upon creation, so be sure to store the returned secret along with the access_id. <B>CREATE OPTIONS</B> The ``create`` sub-command has the following option -p <project> Specify the ID or number of the project in which to create a key. """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The ``hmac delete`` command permanently deletes the specified HMAC key: gsutil hmac delete GOOG56JBMFZX6PMPTQ62VD2 Note that keys must be updated to be in the ``INACTIVE`` state before they can be deleted. <B>DELETE OPTIONS</B> The ``delete`` sub-command has the following option -p <project> Specify the ID or number of the project from which to delete a key. """ _GET_DESCRIPTION = """ <B>GET</B> The ``hmac get`` command retrieves the specified HMAC key's metadata: gsutil hmac get GOOG56JBMFZX6PMPTQ62VD2 Note that there is no option to retrieve a key's secret material after it has been created. <B>GET OPTIONS</B> The ``get`` sub-command has the following option -p <project> Specify the ID or number of the project from which to get a key. """ _LIST_DESCRIPTION = """ <B>LIST</B> The ``hmac list`` command lists the HMAC key metadata for keys in the specified project. If no project is specified in the command, the default project is used. <B>LIST OPTIONS</B> The ``list`` sub-command has the following options -a Show all keys, including recently deleted keys. -l Use long listing format. Shows each key's full metadata excluding the secret. -p <project> Specify the ID or number of the project from which to list keys. -u <service_account_email> Filter keys for a single service account. """ _UPDATE_DESCRIPTION = """ <B>UPDATE</B> The ``hmac update`` command sets the state of the specified key: gsutil hmac update -s INACTIVE -e M42da= GOOG56JBMFZX6PMPTQ62VD2 Valid state arguments are ``ACTIVE`` and ``INACTIVE``. To set a key to state ``DELETED``, use the ``hmac delete`` command on an ``INACTIVE`` key. If an etag is set in the command, it will only succeed if the provided etag matches the etag of the stored key. <B>UPDATE OPTIONS</B> The ``update`` sub-command has the following options -s <ACTIVE\|INACTIVE> Sets the state of the specified key to either ``ACTIVE`` or ``INACTIVE``. -e <etag> If provided, the update will only be performed if the specified etag matches the etag of the stored key. -p <project> Specify the ID or number of the project in which to update a key. """ _SYNOPSIS = (_CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _GET_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _UPDATE_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ You can use the ``hmac`` command to interact with service account `HMAC keys <https://cloud.google.com/storage/docs/authentication/hmackeys>`_. The ``hmac`` command has five sub-commands: """ + '\n'.join([ _CREATE_DESCRIPTION, _DELETE_DESCRIPTION, _GET_DESCRIPTION, _LIST_DESCRIPTION, _UPDATE_DESCRIPTION, ]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _VALID_UPDATE_STATES = ['INACTIVE', 'ACTIVE'] _TIME_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' _create_help_text = CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION) _delete_help_text = CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _list_help_text = CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION) _update_help_text = CreateHelpText(_UPDATE_SYNOPSIS, _UPDATE_DESCRIPTION) def _AccessIdException(command_name, subcommand, synopsis): return CommandException( '%s %s requires an Access ID to be specified as the last argument.\n%s' % (command_name, subcommand, synopsis)) def _KeyMetadataOutput(metadata): """Format the key metadata for printing to the console.""" def FormatInfo(name, value, new_line=True): """Format the metadata name-value pair into two aligned columns.""" width = 22 info_str = '\t%-*s %s' % (width, name + ':', value) if new_line: info_str += '\n' return info_str message = 'Access ID %s:\n' % metadata.accessId message += FormatInfo('State', metadata.state) message += FormatInfo('Service Account', metadata.serviceAccountEmail) message += FormatInfo('Project', metadata.projectId) message += FormatInfo('Time Created', metadata.timeCreated.strftime(_TIME_FORMAT)) message += FormatInfo('Time Last Updated', metadata.updated.strftime(_TIME_FORMAT)) message += FormatInfo('Etag', metadata.etag, new_line=False) return message _CREATE_COMMAND_FORMAT = ('--format=value[separator="' + shim_util.get_format_flag_newline() + '"](' + 'format("Access ID: {}", metadata.accessId),' + 'format("Secret: {}", secret))') _DESCRIBE_COMMAND_FORMAT = ( '--format=value[separator="' + shim_util.get_format_flag_newline() + '"](format("Access ID {}:", accessId),' + 'format("\tState: {}", state),' + 'format("\tService Account: {}", serviceAccountEmail),' + 'format("\tProject: {}", projectId),' + 'format("\tTime Created: {}",' + ' timeCreated.date(format="%a\',\' %d %b %Y %H:%M:%S GMT")),' + 'format("\tTime Last Updated: {}",' + ' updated.date(format="%a\',\' %d %b %Y %H:%M:%S GMT")),' + 'format("\tEtag: {}", etag))') _LIST_COMMAND_SHORT_FORMAT = ( '--format=table[no-heading](format("{}\t{:<12} {}",' 'accessId, state, serviceAccountEmail))') _PROJECT_FLAG = GcloudStorageFlag('--project') CREATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'create', _CREATE_COMMAND_FORMAT], flag_map={ '-p': _PROJECT_FLAG, }) DELETE_COMMAND = GcloudStorageMap(gcloud_command=['storage', 'hmac', 'delete'], flag_map={ '-p': _PROJECT_FLAG, }) GET_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'describe', _DESCRIBE_COMMAND_FORMAT], flag_map={'-p': _PROJECT_FLAG}) LIST_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _LIST_COMMAND_SHORT_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) LIST_COMMAND_LONG_FORMAT = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'list', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-a': GcloudStorageFlag('--all'), '-l': GcloudStorageFlag('--long'), '-u': GcloudStorageFlag('--service-account'), '-p': _PROJECT_FLAG }) UPDATE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'hmac', 'update', _DESCRIBE_COMMAND_FORMAT], flag_map={ '-s': GcloudStorageFlag({ 'ACTIVE': '--activate', 'INACTIVE': '--deactivate', }), '-e': GcloudStorageFlag('--etag'), '-p': _PROJECT_FLAG }) class HmacCommand(Command): """Implementation of gsutil hmac command.""" command_spec = Command.CreateCommandSpec( 'hmac', min_args=1, max_args=8, supported_sub_args='ae:lp:s:u:', file_url_ok=True, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, usage_synopsis=_SYNOPSIS, argparse_arguments={ 'create': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'delete': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'get': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'list': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], 'update': [CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()], }, ) help_spec = Command.HelpSpec( help_name='hmac', help_name_aliases=[], help_type='command_help', help_one_line_summary=('CRUD operations on service account HMAC keys.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'delete': _delete_help_text, 'get': _get_help_text, 'list': _list_help_text, 'update': _update_help_text, }) def get_gcloud_storage_args(self): if self.args[0] == 'list' and '-l' in self.args: gcloud_storage_map = GcloudStorageMap( gcloud_command={'list': LIST_COMMAND_LONG_FORMAT}, flag_map={}, ) else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': CREATE_COMMAND, 'delete': DELETE_COMMAND, 'update': UPDATE_COMMAND, 'get': GET_COMMAND, 'list': LIST_COMMAND }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _CreateHmacKey(self, thread_state=None): """Creates HMAC key for a service account.""" if self.args: self.service_account_email = self.args[0] else: err_msg = ('%s %s requires a service account to be specified as the ' 'last argument.\n%s') raise CommandException( err_msg % (self.command_name, self.action_subcommand, _CREATE_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.CreateHmacKey(self.project_id, self.service_account_email, provider='gs') print('%-12s %s' % ('Access ID:', response.metadata.accessId)) print('%-12s %s' % ('Secret:', response.secret)) def _DeleteHmacKey(self, thread_state=None): """Deletes an HMAC key.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _DELETE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) gsutil_api.DeleteHmacKey(self.project_id, access_id, provider='gs') def _GetHmacKey(self, thread_state=None): """Gets HMAC key from its Access Id.""" if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _GET_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.GetHmacKey(self.project_id, access_id, provider='gs') print(_KeyMetadataOutput(response)) def _ListHmacKeys(self, thread_state=None): """Lists HMAC keys for a project or service account.""" if self.args: raise CommandException( '%s %s received unexpected arguments.\n%s' % (self.command_name, self.action_subcommand, _LIST_SYNOPSIS)) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.ListHmacKeys(self.project_id, self.service_account_email, self.show_all, provider='gs') short_list_format = '%s\t%-12s %s' if self.long_list: for item in response: print(_KeyMetadataOutput(item)) print() else: for item in response: print(short_list_format % (item.accessId, item.state, item.serviceAccountEmail)) def _UpdateHmacKey(self, thread_state=None): """Update an HMAC key's state.""" if not self.state: raise CommandException( 'A state flag must be supplied for %s %s\n%s' % (self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS)) elif self.state not in _VALID_UPDATE_STATES: raise CommandException('The state flag value must be one of %s' % ', '.join(_VALID_UPDATE_STATES)) if self.args: access_id = self.args[0] else: raise _AccessIdException(self.command_name, self.action_subcommand, _UPDATE_SYNOPSIS) gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) response = gsutil_api.UpdateHmacKey(self.project_id, access_id, self.state, self.etag, provider='gs') print(_KeyMetadataOutput(response)) def RunCommand(self): """Command entry point for the hmac command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException( 'The "hmac" command can only be used with the GCS JSON API') self.action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.service_account_email = None self.state = None self.show_all = False self.long_list = False self.etag = None if self.sub_opts: for o, a in self.sub_opts: if o == '-u': self.service_account_email = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-s': self.state = a elif o == '-a': self.show_all = True elif o == '-l': self.long_list = True elif o == '-e': self.etag = a if not self.project_id: self.project_id = PopulateProjectId(None) method_for_arg = { 'create': self._CreateHmacKey, 'delete': self._DeleteHmacKey, 'get': self._GetHmacKey, 'list': self._ListHmacKeys, 'update': self._UpdateHmacKey, } if self.action_subcommand not in method_for_arg: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help hmac".' % (self.action_subcommand, self.command_name)) LogCommandParams(subcommands=[self.action_subcommand]) method_for_arg[self.action_subcommand]() return 0	16.66 KB	2025-06-25 08:19
📄 iam.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of IAM policy management command for GCS.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import json import os import re import subprocess import textwrap import six from six.moves import zip from apitools.base.protorpclite import protojson from apitools.base.protorpclite.messages import DecodeError from boto import config from gslib.cloud_api import ArgumentException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import GetFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import IamChOnResourceWithConditionsException from gslib.help_provider import CreateHelpText from gslib.metrics import LogCommandParams from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.storage_url import GetSchemeFromUrlString from gslib.storage_url import IsKnownUrlScheme from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import IAM_POLICY_VERSION from gslib.utils.constants import NO_MAX from gslib.utils import iam_helper from gslib.utils.iam_helper import BindingStringToTuple from gslib.utils.iam_helper import BindingsTuple from gslib.utils.iam_helper import DeserializeBindingsTuple from gslib.utils.iam_helper import IsEqualBindings from gslib.utils.iam_helper import PatchBindings from gslib.utils.iam_helper import SerializeBindingsTuple from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.shim_util import GcloudStorageFlag _SET_SYNOPSIS = """ gsutil iam set [-afRr] [-e <etag>] file url ... """ _GET_SYNOPSIS = """ gsutil iam get url """ # Note that the commands below are put in quotation marks instead of backticks; # this is done because the whole ch synopsis gets rendered in one <pre> tag in # the web docs, and having literal double-backticks looks weird. _CH_SYNOPSIS = """ gsutil iam ch [-fRr] binding ... url where each binding is of the form: [-d] ("user"\|"serviceAccount"\|"domain"\|"group"):id:role[,...] [-d] ("allUsers"\|"allAuthenticatedUsers"):role[,...] -d ("user"\|"serviceAccount"\|"domain"\|"group"):id -d ("allUsers"\|"allAuthenticatedUsers") NOTE: The "iam ch" command does not support changing Cloud IAM policies with bindings that contain conditions. As such, "iam ch" cannot be used to add conditions to a policy or to change the policy of a resource that already contains conditions. See additional details below. NOTE: The "gsutil iam" command does not allow you to add convenience values (projectOwner, projectEditor, projectViewer), but you can remove existing ones. """ _GET_DESCRIPTION = """ <B>GET</B> The ``iam get`` command gets the Cloud IAM policy for a bucket or object, which you can save and edit for use with the ``iam set`` command. The following examples save the bucket or object's Cloud IAM policy to a text file: gsutil iam get gs://example > bucket_iam.txt gsutil iam get gs://example/important.txt > object_iam.txt The Cloud IAM policy returned by ``iam get`` includes an etag. The etag is used in the precondition check for ``iam set`` unless you override it using ``iam set -e``. """ _SET_DESCRIPTION = """ <B>SET</B> The ``iam set`` command sets a Cloud IAM policy on one or more buckets or objects, replacing the existing policy on those buckets or objects. For an example of the correct formatting for a Cloud IAM policy, see the output of the ``iam get`` command. You can use the ``iam ch`` command to edit an existing policy, even in the presence of concurrent updates. You can also edit the policy concurrently using the ``-e`` flag to override the Cloud IAM policy's etag. Specifying ``-e`` with an empty string (i.e. ``gsutil iam set -e '' ...``) instructs gsutil to skip the precondition check when setting the Cloud IAM policy. When you set a Cloud IAM policy on a large number of objects, you should use the gsutil ``-m`` option for concurrent processing. The following command applies ``iam.txt`` to all objects in the ``dogs`` bucket: gsutil -m iam set -r iam.txt gs://dogs Note that only object-level operations are parallelized; setting a Cloud IAM policy on a large number of buckets with the ``-m`` flag does not improve performance. <B>SET OPTIONS</B> The ``set`` sub-command has the following options: -R, -r Performs ``iam set`` recursively on all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -a Performs ``iam set`` on all object versions. -e <etag> Performs the precondition check on each object with the specified etag before setting the policy. You can retrieve the policy's etag using ``iam get``. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This option is implicitly set when you use the gsutil ``-m`` option. """ _CH_DESCRIPTION = """ <B>CH</B> The ``iam ch`` command incrementally updates Cloud IAM policies. You can specify multiple access grants or removals in a single command. The access changes are applied as a batch to each url in the order in which they appear in the command line arguments. Each access change specifies a principal and a role that is either granted or revoked. You can use gsutil ``-m`` to handle object-level operations in parallel. NOTE: The ``iam ch`` command cannot be used to change the Cloud IAM policy of a resource that contains conditions in its policy bindings. Attempts to do so result in an error. To change the Cloud IAM policy of such a resource, you can perform a read-modify-write operation by saving the policy to a file using ``iam get``, editing the file, and setting the updated policy using ``iam set``. <B>CH EXAMPLES</B> Examples for the ``ch`` sub-command: To grant a single role to a single principal for some targets: gsutil iam ch user:john.doe@example.com:objectCreator gs://ex-bucket To make a bucket's objects publicly readable: gsutil iam ch allUsers:objectViewer gs://ex-bucket To grant multiple bindings to a bucket: gsutil iam ch user:john.doe@example.com:objectCreator \\ domain:www.my-domain.org:objectViewer gs://ex-bucket To specify more than one role for a particular principal: gsutil iam ch user:john.doe@example.com:objectCreator,objectViewer \\ gs://ex-bucket To specify a custom role for a particular principal: gsutil iam ch user:john.doe@example.com:roles/customRoleName gs://ex-bucket To apply a grant and simultaneously remove a binding to a bucket: gsutil iam ch -d group:readers@example.com:legacyBucketReader \\ group:viewers@example.com:objectViewer gs://ex-bucket To remove a user from all roles on a bucket: gsutil iam ch -d user:john.doe@example.com gs://ex-bucket <B>CH OPTIONS</B> The ``ch`` sub-command has the following options: -d Removes roles granted to the specified principal. -R, -r Performs ``iam ch`` recursively to all objects under the specified bucket. This flag can only be set if the policy exclusively uses ``roles/storage.legacyObjectReader`` or ``roles/storage.legacyObjectOwner``. This flag cannot be used if the bucket is configured for uniform bucket-level access. -f The default gsutil error-handling mode is fail-fast. This flag changes the request to fail-silent mode. This is implicitly set when you invoke the gsutil ``-m`` option. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Cloud Identity and Access Management (Cloud IAM) allows you to control who has access to the resources in your Google Cloud project. For more information, see `Cloud Identity and Access Management <https://cloud.google.com/storage/docs/access-control/iam>`_. The iam command has three sub-commands: """ + '\n'.join([_GET_DESCRIPTION, _SET_DESCRIPTION, _CH_DESCRIPTION]) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) STORAGE_URI_REGEX = re.compile(r'[a-z]+://.+') IAM_CH_CONDITIONS_WORKAROUND_MSG = ( 'To change the IAM policy of a resource that has bindings containing ' 'conditions, perform a read-modify-write operation using "iam get" and ' '"iam set".') def _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(urls, recursion_requested): if UrlsAreMixOfBucketsAndObjects(urls) and not recursion_requested: raise CommandException('Cannot operate on a mix of buckets and objects.') def _PatchIamWrapper(cls, iter_result, thread_state): (serialized_bindings_tuples, expansion_result) = iter_result return cls.PatchIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. [DeserializeBindingsTuple(t) for t in serialized_bindings_tuples], thread_state=thread_state) def _SetIamWrapper(cls, iter_result, thread_state): (serialized_policy, expansion_result) = iter_result return cls.SetIamHelper( expansion_result.expanded_storage_url, # Deserialize the JSON object passed from Command.Apply. protojson.decode_message(apitools_messages.Policy, serialized_policy), thread_state=thread_state) def _SetIamExceptionHandler(cls, e): cls.logger.error(str(e)) def _PatchIamExceptionHandler(cls, e): cls.logger.error(str(e)) class IamCommand(Command): """Implementation of gsutil iam command.""" command_spec = Command.CreateCommandSpec( 'iam', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='afRrd:e:', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'ch': [ CommandArgument.MakeOneOrMoreBindingsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], }, ) help_spec = Command.HelpSpec( help_name='iam', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change bucket and/or object IAM permissions.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) def _get_resource_type(self, url_patterns): if self.recursion_requested or url_patterns[0].IsObject(): return 'objects' else: return 'buckets' def get_gcloud_storage_args(self): self.sub_command = self.args.pop(0) if self.sub_command == 'get': if StorageUrlFromString(self.args[0]).IsObject(): command_group = 'objects' else: command_group = 'buckets' gcloud_storage_map = GcloudStorageMap(gcloud_command=[ 'storage', command_group, 'get-iam-policy', '--format=json' ], flag_map={}) elif self.sub_command == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', None, 'set-iam-policy', '--format=json'], flag_map={ '-a': GcloudStorageFlag('--all-versions'), '-e': GcloudStorageFlag('--etag'), # Hack to make gcloud recognize empty string value. '_empty_etag': GcloudStorageFlag('--etag='), '-f': GcloudStorageFlag('--continue-on-error'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), }) # Flags must be at the start of self.args to get parsed. self.ParseSubOpts() # Flags moved to self.sub_opts, leaving [POLICY-FILE, URLS...]. url_strings = self.args[1:] url_objects = list(map(StorageUrlFromString, url_strings)) recurse = False for i, (flag_key, flag_value) in enumerate(self.sub_opts): if flag_key in ('-r', '-R'): recurse = True elif flag_key == '-e' and flag_value == '': # Allow for empty string values. self.sub_opts[i] = ('_empty_etag', '') _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, recurse) if recurse or url_objects[0].IsObject(): gcloud_storage_map.gcloud_command[1] = 'objects' else: gcloud_storage_map.gcloud_command[1] = 'buckets' # Gsutil takes policy file then URLs, and gcloud storage does the reverse. self.args = url_strings + self.args[:1] elif self.sub_command == 'ch': if shim_util.HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) is shim_util.HIDDEN_SHIM_MODE.DRY_RUN: self.logger.warning( 'The shim maps iam ch commands to several gcloud storage commands,' ' which cannot be determined without running gcloud storage.') return [] # Translation occurs in self.run_gcloud_storage. return super().get_gcloud_storage_args(gcloud_storage_map) def _RaiseIfInvalidUrl(self, url): if url.IsFileUrl(): error_msg = 'Invalid Cloud URL "%s".' % url.object_name if set(url.object_name).issubset(set('-Rrf')): error_msg += ( ' This resource handle looks like a flag, which must appear' ' before all bindings. See "gsutil help iam ch" for more details.') raise CommandException(error_msg) def _GetSettingsAndDiffs(self): self.continue_on_error = False self.recursion_requested = False patch_bindings_tuples = [] if self.sub_opts: for o, a in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-d': patch_bindings_tuples.append(BindingStringToTuple(False, a)) url_pattern_strings = [] # N.B.: self.sub_opts stops taking in options at the first non-flagged # token. The rest of the tokens are sent to self.args. Thus, in order to # handle input of the form "-d <binding> <binding> <url>", we will have to # parse self.args for a mix of both bindings and CloudUrls. We are not # expecting to come across the -r, -f flags here. it = iter(self.args) for token in it: if (STORAGE_URI_REGEX.match(token) and IsKnownUrlScheme(GetSchemeFromUrlString(token))): url_pattern_strings.append(token) break if token == '-d': try: patch_bindings_tuples.append(BindingStringToTuple(False, next(it))) except StopIteration: raise CommandException( 'A -d flag is missing an argument specifying bindings to remove.') else: patch_bindings_tuples.append(BindingStringToTuple(True, token)) if not patch_bindings_tuples: raise CommandException('Must specify at least one binding.') # All following arguments are urls. for token in it: url_pattern_strings.append(token) url_objects = list(map(StorageUrlFromString, url_pattern_strings)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(url_objects, self.recursion_requested) return patch_bindings_tuples, url_objects def _run_ch_subprocess(self, args, stdin=None): _, command = self._get_full_gcloud_storage_execution_information(args) process = subprocess.run( command, env=self._get_shim_command_environment_variables(), input=stdin, stderr=subprocess.PIPE, stdout=subprocess.PIPE, text=True, ) if process.returncode != 0: if self.continue_on_error: self.logger.error(process.stderr) else: raise CommandException(process.stderr) return process def run_gcloud_storage(self): if self.sub_command != 'ch': return super().run_gcloud_storage() self.ParseSubOpts() bindings_tuples, patterns = self._GetSettingsAndDiffs() resource_type = self._get_resource_type(patterns) list_settings = [] if self.recursion_requested: list_settings.append('-r') return_code = 0 for url_pattern in patterns: self._RaiseIfInvalidUrl(url_pattern) if resource_type == 'objects': ls_process = self._run_ch_subprocess(['storage', 'ls', '--json'] + list_settings + [str(url_pattern)]) if ls_process.returncode: return_code = 1 continue ls_output = json.loads(ls_process.stdout) urls = [ resource['url'] for resource in ls_output if resource['type'] == 'cloud_object' ] else: urls = [str(url_pattern)] for url in urls: get_process = self._run_ch_subprocess( ['storage', resource_type, 'get-iam-policy', url, '--format=json']) if get_process.returncode: return_code = 1 continue policy = json.loads(get_process.stdout) bindings = iam_helper.BindingsDictToUpdateDict(policy['bindings']) for (is_grant, diff) in bindings_tuples: diff_dict = iam_helper.BindingsDictToUpdateDict(diff) bindings = PatchBindings(bindings, diff_dict, is_grant) policy['bindings'] = [{ 'role': r, 'members': sorted(list(m)) } for r, m in sorted(six.iteritems(bindings))] set_process = self._run_ch_subprocess( ['storage', resource_type, 'set-iam-policy', url, '-'], stdin=json.dumps(policy, sort_keys=True)) if set_process.returncode: return_code = 1 return return_code def GetIamHelper(self, storage_url, thread_state=None): """Gets an IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the global -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Returns: Policy instance. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): policy = gsutil_api.GetBucketIamPolicy( storage_url.bucket_name, provider=storage_url.scheme, fields=['bindings', 'etag'], ) else: policy = gsutil_api.GetObjectIamPolicy( storage_url.bucket_name, storage_url.object_name, generation=storage_url.generation, provider=storage_url.scheme, fields=['bindings', 'etag'], ) return policy def _GetIam(self, thread_state=None): """Gets IAM policy for single bucket or object.""" pattern = self.args[0] matches = PluralityCheckableIterator( self.WildcardIterator(pattern).IterAll(bucket_listing_fields=['name'])) if matches.IsEmpty(): raise CommandException('%s matched no URLs' % pattern) if matches.HasPlurality(): raise CommandException( '%s matched more than one URL, which is not allowed by the %s ' 'command' % (pattern, self.command_name)) storage_url = StorageUrlFromString(list(matches)[0].url_string) policy = self.GetIamHelper(storage_url, thread_state=thread_state) policy_json = json.loads(protojson.encode_message(policy)) policy_str = json.dumps( policy_json, sort_keys=True, separators=(',', ': '), indent=2, ) print(policy_str) def _SetIamHelperInternal(self, storage_url, policy, thread_state=None): """Sets IAM policy for a single, resolved bucket / object URL. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. policy: A Policy object to set on the bucket / object. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. Raises: ServiceException passed from the API call if an HTTP error was returned. """ # SetIamHelper may be called by a command.WorkerThread. In the # single-threaded case, WorkerThread will not pass the CloudApiDelegator # instance to thread_state. GetCloudInstance is called to resolve the # edge case. gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) if storage_url.IsBucket(): gsutil_api.SetBucketIamPolicy(storage_url.bucket_name, policy, provider=storage_url.scheme) else: gsutil_api.SetObjectIamPolicy(storage_url.bucket_name, storage_url.object_name, policy, generation=storage_url.generation, provider=storage_url.scheme) def SetIamHelper(self, storage_url, policy, thread_state=None): """Handles the potential exception raised by the internal set function.""" try: self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise def PatchIamHelper(self, storage_url, bindings_tuples, thread_state=None): """Patches an IAM policy for a single, resolved bucket / object URL. The patch is applied by altering the policy from an IAM get request, and setting the new IAM with the specified etag. Because concurrent IAM set requests may alter the etag, we may need to retry this operation several times before success. Args: storage_url: A CloudUrl instance with no wildcards, pointing to a specific bucket or object. bindings_tuples: A list of BindingsTuple instances. thread_state: CloudApiDelegator instance which is passed from command.WorkerThread.__init__() if the -m flag is specified. Will use self.gsutil_api if thread_state is set to None. """ try: self._PatchIamHelperInternal(storage_url, bindings_tuples, thread_state=thread_state) except ServiceException: if self.continue_on_error: self.everything_set_okay = False else: raise except IamChOnResourceWithConditionsException as e: if self.continue_on_error: self.everything_set_okay = False self.tried_ch_on_resource_with_conditions = True self.logger.debug(e.message) else: raise CommandException(e.message) @Retry(PreconditionException, tries=3, timeout_secs=1.0) def _PatchIamHelperInternal(self, storage_url, bindings_tuples, thread_state=None): policy = self.GetIamHelper(storage_url, thread_state=thread_state) (etag, bindings) = (policy.etag, policy.bindings) # If any of the bindings have conditions present, raise an exception. # See the docstring for the IamChOnResourceWithConditionsException class # for more details on why we raise this exception. for binding in bindings: if binding.condition: message = 'Could not patch IAM policy for %s.' % storage_url message += '\n' message += '\n'.join( textwrap.wrap( 'The resource had conditions present in its IAM policy bindings, ' 'which is not supported by "iam ch". %s' % IAM_CH_CONDITIONS_WORKAROUND_MSG)) raise IamChOnResourceWithConditionsException(message) # Create a backup which is untainted by any references to the original # bindings. orig_bindings = list(bindings) for (is_grant, diff) in bindings_tuples: bindings_dict = iam_helper.BindingsMessageToUpdateDict(bindings) diff_dict = iam_helper.BindingsMessageToUpdateDict(diff) new_bindings_dict = PatchBindings(bindings_dict, diff_dict, is_grant) bindings = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(new_bindings_dict) ] if IsEqualBindings(bindings, orig_bindings): self.logger.info('No changes made to %s', storage_url) return policy = apitools_messages.Policy(bindings=bindings, etag=etag) # We explicitly wish for etag mismatches to raise an error and allow this # function to error out, so we are bypassing the exception handling offered # by IamCommand.SetIamHelper in lieu of our own handling (@Retry). self._SetIamHelperInternal(storage_url, policy, thread_state=thread_state) def _PatchIam(self): raw_bindings_tuples, url_patterns = self._GetSettingsAndDiffs() patch_bindings_tuples = [] for is_grant, bindings in raw_bindings_tuples: bindings_messages = [] for binding in bindings: bindings_message = apitools_messages.Policy.BindingsValueListEntry( members=binding['members'], role=binding['role']) bindings_messages.append(bindings_message) patch_bindings_tuples.append( BindingsTuple(is_grant=is_grant, bindings=bindings_messages)) self.everything_set_okay = True self.tried_ch_on_resource_with_conditions = False threaded_wildcards = [] for surl in url_patterns: try: if surl.IsBucket(): if self.recursion_requested: surl.object = '' threaded_wildcards.append(surl.url_string) else: self.PatchIamHelper(surl, patch_bindings_tuples) else: threaded_wildcards.append(surl.url_string) except AttributeError: self._RaiseIfInvalidUrl(surl) if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) serialized_bindings_tuples_it = itertools.repeat( [SerializeBindingsTuple(t) for t in patch_bindings_tuples]) self.Apply(_PatchIamWrapper, zip(serialized_bindings_tuples_it, name_expansion_iterator), _PatchIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: msg = 'Some IAM policies could not be patched.' if self.tried_ch_on_resource_with_conditions: msg += '\n' msg += '\n'.join( textwrap.wrap( 'Some resources had conditions present in their IAM policy ' 'bindings, which is not supported by "iam ch". %s' % (IAM_CH_CONDITIONS_WORKAROUND_MSG))) raise CommandException(msg) # TODO(iam-beta): Add an optional flag to specify etag and edit the policy # accordingly to be passed into the helper functions. def _SetIam(self): """Set IAM policy for given wildcards on the command line.""" self.continue_on_error = False self.recursion_requested = False self.all_versions = False force_etag = False etag = '' if self.sub_opts: for o, arg in self.sub_opts: if o in ['-r', '-R']: self.recursion_requested = True elif o == '-f': self.continue_on_error = True elif o == '-a': self.all_versions = True elif o == '-e': etag = str(arg) force_etag = True else: self.RaiseInvalidArgumentException() file_url = self.args[0] patterns = self.args[1:] # Load the IAM policy file and raise error if the file is invalid JSON or # does not exist. try: with open(file_url, 'r') as fp: policy = json.loads(fp.read()) except IOError: raise ArgumentException('Specified IAM policy file "%s" does not exist.' % file_url) except ValueError as e: self.logger.debug('Invalid IAM policy file, ValueError:\n%s', e) raise ArgumentException('Invalid IAM policy file "%s".' % file_url) bindings = policy.get('bindings', []) if not force_etag: etag = policy.get('etag', '') policy_json = json.dumps({ 'bindings': bindings, 'etag': etag, 'version': IAM_POLICY_VERSION }) try: policy = protojson.decode_message(apitools_messages.Policy, policy_json) except DecodeError: raise ArgumentException('Invalid IAM policy file "%s" or etag "%s".' % (file_url, etag)) self.everything_set_okay = True # This list of wildcard strings will be handled by NameExpansionIterator. threaded_wildcards = [] surls = list(map(StorageUrlFromString, patterns)) _RaiseErrorIfUrlsAreMixOfBucketsAndObjects(surls, self.recursion_requested) for surl in surls: print(surl.url_string) if surl.IsBucket(): if self.recursion_requested: surl.object_name = '' threaded_wildcards.append(surl.url_string) else: self.SetIamHelper(surl, policy) else: threaded_wildcards.append(surl.url_string) # N.B.: If threaded_wildcards contains a non-existent bucket # (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator # will raise an exception in iter.next. This halts all iteration, even # when -f is set. This behavior is also evident in acl set. This behavior # also appears for any exception that will be raised when iterating over # wildcard expansions (access denied if bucket cannot be listed, etc.). if threaded_wildcards: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, threaded_wildcards, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), threaded_wildcards, self.recursion_requested, all_versions=self.all_versions) policy_it = itertools.repeat(protojson.encode_message(policy)) self.Apply(_SetIamWrapper, zip(policy_it, name_expansion_iterator), _SetIamExceptionHandler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) self.everything_set_okay &= not GetFailureCount() > 0 # TODO: Add an error counter for files and objects. if not self.everything_set_okay: raise CommandException('Some IAM policies could not be set.') def RunCommand(self): """Command entry point for the acl command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. LogCommandParams(sub_opts=self.sub_opts) self.def_acl = False if action_subcommand == 'get': LogCommandParams(subcommands=[action_subcommand]) self._GetIam() elif action_subcommand == 'set': LogCommandParams(subcommands=[action_subcommand]) self._SetIam() elif action_subcommand == 'ch': LogCommandParams(subcommands=[action_subcommand]) self._PatchIam() else: raise CommandException('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help iam".' % (action_subcommand, self.command_name)) return 0	34 KB	2025-06-25 08:19
📄 kms.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the kms command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import textwrap from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.kms_api import KmsApi from gslib.project_id import PopulateProjectId from gslib.third_party.kms_apitools.cloudkms_v1_messages import Binding from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.constants import NO_MAX from gslib.utils.encryption_helper import ValidateCMEK from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _AUTHORIZE_SYNOPSIS = """ gsutil kms authorize [-p <proj_id>] -k <kms_key> """ _ENCRYPTION_SYNOPSIS = """ gsutil kms encryption [(-d\|[-k <kms_key>])] [-w] gs://<bucket_name>... """ _SERVICEACCOUNT_SYNOPSIS = """ gsutil kms serviceaccount [-p <proj_id>] """ _SYNOPSIS = (_AUTHORIZE_SYNOPSIS + _ENCRYPTION_SYNOPSIS.lstrip('\n') + _SERVICEACCOUNT_SYNOPSIS.lstrip('\n') + '\n') # pylint: disable=line-too-long _AUTHORIZE_DESCRIPTION = """ <B>AUTHORIZE</B> The authorize sub-command checks that the default (or supplied) project has a Cloud Storage service agent created for it, and if not, it creates one. It then adds appropriate encrypt/decrypt permissions to Cloud KMS resources such that the service agent can write and read Cloud KMS-encrypted objects in buckets associated with the service agent's project. <B>AUTHORIZE EXAMPLES</B> Authorize "my-project" to use a Cloud KMS key: gsutil kms authorize -p my-project \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key <B>AUTHORIZE OPTIONS</B> -k <key> The path to the KMS key to use. The path has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -p <project> The ID or number of the project being authorized to use the Cloud KMS key. If this flag is not included, your default project is authorized. """ _ENCRYPTION_DESCRIPTION = """ <B>ENCRYPTION</B> The encryption sub-command is used to set, display, or clear a bucket's default KMS key, which is used to encrypt newly-written objects if no other key is specified. <B>ENCRYPTION EXAMPLES</B> Set the default KMS key for my-bucket: gsutil kms encryption \\ -k projects/key-project/locations/us-east1/keyRings/key-ring/cryptoKeys/my-key \\ gs://my-bucket Show the default KMS key for my-bucket, if one is set: gsutil kms encryption gs://my-bucket Clear the default KMS key so newly-written objects are not encrypted using it: gsutil kms encryption -d gs://my-bucket Once you clear the default KMS key, newly-written objects are encrypted with Google-managed encryption keys by default. <B>ENCRYPTION OPTIONS</B> -k <key> Set the default KMS key for my-bucket using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -w (used with -k key) Display a warning rather than failing if gsutil is unable to verify that the specified key contains the correct IAM bindings for encryption/decryption. This is useful for users that do not have getIamPolicy permission but know that the key has the correct IAM policy for encryption in the user's project. -d Clear the default KMS key. """ # pylint: enable=line-too-long _SERVICEACCOUNT_DESCRIPTION = """ <B>SERVICEACCOUNT</B> The serviceaccount sub-command displays the Cloud Storage service agent that is used to perform Cloud KMS operations against your default project (or a supplied project). <B>SERVICEACCOUNT EXAMPLES</B> Show the service account for my-project: gsutil kms serviceaccount -p my-project <B>SERVICEACCOUNT OPTIONS</B> -p <project> The ID or number of the project whose Cloud Storage service agent is being requested. If this flag is not included, your default project is used. """ _DESCRIPTION = """ The kms command is used to configure Google Cloud Storage and Cloud KMS resources to support encryption of Cloud Storage objects with `Cloud KMS keys <https://cloud.google.com/storage/docs/encryption/customer-managed-keys>`_. The kms command has three sub-commands that deal with configuring Cloud Storage's integration with Cloud KMS: ``authorize``, ``encryption``, and ``serviceaccount``. Before using this command, read the `prerequisites <https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys#prereqs>`_. for using Cloud KMS with Cloud Storage. """ + (_AUTHORIZE_DESCRIPTION + _ENCRYPTION_DESCRIPTION + _SERVICEACCOUNT_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _authorize_help_text = CreateHelpText(_AUTHORIZE_SYNOPSIS, _AUTHORIZE_DESCRIPTION) _encryption_help_text = CreateHelpText(_ENCRYPTION_SYNOPSIS, _ENCRYPTION_DESCRIPTION) _serviceaccount_help_text = CreateHelpText(_SERVICEACCOUNT_SYNOPSIS, _SERVICEACCOUNT_DESCRIPTION) _AUTHORIZE_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), '-k': GcloudStorageFlag('--authorize-cmek'), }) _SERVICEACCOUNT_COMMAND = GcloudStorageMap( gcloud_command=['storage', 'service-agent'], flag_map={ '-p': GcloudStorageFlag('--project'), }) class KmsCommand(Command): """Implements of gsutil kms command.""" command_spec = Command.CreateCommandSpec( 'kms', usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='dk:p:w', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'authorize': [], 'encryption': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'serviceaccount': [], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='kms', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure Cloud KMS encryption', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'authorize': _authorize_help_text, 'encryption': _encryption_help_text, 'serviceaccount': _serviceaccount_help_text }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'authorize': _AUTHORIZE_COMMAND, 'serviceaccount': _SERVICEACCOUNT_COMMAND, # "encryption" subcommand handled in get_gcloud_storage_args. }, flag_map={}) def get_gcloud_storage_args(self): if self.args[0] == 'encryption': gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'encryption': GcloudStorageMap( gcloud_command=['storage', 'buckets'], flag_map={ '-d': GcloudStorageFlag('--clear-default-encryption-key'), '-k': GcloudStorageFlag('--default-encryption-key'), '-w': GcloudStorageFlag(''), }), }, flag_map={}) if '-d' in self.args or '-k' in self.args: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'update' ] else: gcloud_storage_map.gcloud_command['encryption'].gcloud_command += [ 'describe', ('--format=value[separator=\": \"](name, encryption' '.defaultKmsKeyName.yesno(no="No default encryption key."))'), '--raw' ] else: gcloud_storage_map = KmsCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GatherSubOptions(self, subcommand_name): self.CheckArguments() self.clear_kms_key = False self.kms_key = None self.warn_on_key_authorize_failure = False if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a elif o == '-k': self.kms_key = a ValidateCMEK(self.kms_key) elif o == '-d': self.clear_kms_key = True elif o == '-w': self.warn_on_key_authorize_failure = True if self.warn_on_key_authorize_failure and ( self.subcommand_name != 'encryption' or not self.kms_key): raise CommandException('\n'.join( textwrap.wrap( 'The "-w" option should only be specified for the "encryption" ' 'subcommand and must be used with the "-k" option.'))) # Determine the project (used in the serviceaccount and authorize # subcommands), either from the "-p" option's value or the default specified # in the user's Boto config file. if not self.project_id: self.project_id = PopulateProjectId(None) @Retry(ServiceException, tries=3, timeout_secs=1) def _AuthorizeProject(self, project_id, kms_key): """Authorizes a project's service account to be used with a KMS key. Authorizes the Cloud Storage-owned service account for project_id to be used with kms_key. Args: project_id: (str) Project id string (not number). kms_key: (str) Fully qualified resource name for the KMS key. Returns: (str, bool) A 2-tuple consisting of: 1) The email address for the service account associated with the project, which is authorized to encrypt/decrypt with the specified key. 2) A bool value - True if we had to grant the service account permission to encrypt/decrypt with the given key; False if the required permission was already present. """ # Request the Cloud Storage-owned service account for project_id, creating # it if it does not exist. service_account = self.gsutil_api.GetProjectServiceAccount( project_id, provider='gs').email_address kms_api = KmsApi(logger=self.logger) self.logger.debug('Getting IAM policy for %s', kms_key) try: policy = kms_api.GetKeyIamPolicy(kms_key) self.logger.debug('Current policy is %s', policy) # Check if the required binding is already present; if not, add it and # update the key's IAM policy. added_new_binding = False binding = Binding(role='roles/cloudkms.cryptoKeyEncrypterDecrypter', members=['serviceAccount:%s' % service_account]) if binding not in policy.bindings: policy.bindings.append(binding) kms_api.SetKeyIamPolicy(kms_key, policy) added_new_binding = True return (service_account, added_new_binding) except AccessDeniedException: if self.warn_on_key_authorize_failure: text_util.print_to_fd('\n'.join( textwrap.wrap( 'Warning: Check that your Cloud Platform project\'s service ' 'account has the "cloudkms.cryptoKeyEncrypterDecrypter" role ' 'for the specified key. Without this role, you may not be ' 'able to encrypt or decrypt objects using the key which will ' 'prevent you from uploading or downloading objects.'))) return (service_account, False) else: raise def _Authorize(self): self._GatherSubOptions('authorize') if not self.kms_key: raise CommandException('%s %s requires a key to be specified with -k' % (self.command_name, self.subcommand_name)) _, newly_authorized = self._AuthorizeProject(self.project_id, self.kms_key) if newly_authorized: print('Authorized project %s to encrypt and decrypt with key:\n%s' % (self.project_id, self.kms_key)) else: print('Project %s was already authorized to encrypt and decrypt with ' 'key:\n%s.' % (self.project_id, self.kms_key)) return 0 def _EncryptionClearKey(self, bucket_metadata, bucket_url): """Clears the defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. """ bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue() print('Clearing default encryption key for %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _EncryptionSetKey(self, bucket_metadata, bucket_url, svc_acct_for_project_num): """Sets defaultKmsKeyName on a Cloud Storage bucket. Args: bucket_metadata: (apitools_messages.Bucket) Metadata for the given bucket. bucket_url: (gslib.storage_url.StorageUrl) StorageUrl of the given bucket. svc_acct_for_project_num: (Dict[int, str]) Mapping of project numbers to their corresponding service account. """ bucket_project_number = bucket_metadata.projectNumber try: # newly_authorized will always be False if the project number is in our # cache dict, since we've already called _AuthorizeProject on it. service_account, newly_authorized = ( svc_acct_for_project_num[bucket_project_number], False) except KeyError: service_account, newly_authorized = self._AuthorizeProject( bucket_project_number, self.kms_key) svc_acct_for_project_num[bucket_project_number] = service_account if newly_authorized: text_util.print_to_fd('Authorized service account %s to use key:\n%s' % (service_account, self.kms_key)) bucket_metadata.encryption = apitools_messages.Bucket.EncryptionValue( defaultKmsKeyName=self.kms_key) print('Setting default KMS key for bucket %s...' % str(bucket_url).rstrip('/')) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['encryption'], provider=bucket_url.scheme) def _Encryption(self): self._GatherSubOptions('encryption') # For each project, we should only make one API call to look up its # associated Cloud Storage-owned service account; subsequent lookups can be # pulled from this cache dict. svc_acct_for_project_num = {} def _EncryptionForBucket(blr): """Set, clear, or get the defaultKmsKeyName for a bucket.""" bucket_url = blr.storage_url if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) # Determine the project from the provided bucket. bucket_metadata = self.gsutil_api.GetBucket( bucket_url.bucket_name, fields=['encryption', 'projectNumber'], provider=bucket_url.scheme) # "-d" flag was specified, so clear the default KMS key and return. if self.clear_kms_key: self._EncryptionClearKey(bucket_metadata, bucket_url) return 0 # "-k" flag was specified, so set the default KMS key and return. if self.kms_key: self._EncryptionSetKey(bucket_metadata, bucket_url, svc_acct_for_project_num) return 0 # Neither "-d" nor "-k" was specified, so emit the default KMS key and # return. bucket_url_string = str(bucket_url).rstrip('/') if (bucket_metadata.encryption and bucket_metadata.encryption.defaultKmsKeyName): print('Default encryption key for %s:\n%s' % (bucket_url_string, bucket_metadata.encryption.defaultKmsKeyName)) else: print('Bucket %s has no default encryption key' % bucket_url_string) return 0 # Iterate over bucket args, performing the specified encryption operation # for each. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _EncryptionForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _ServiceAccount(self): self.CheckArguments() if not self.args: self.args = ['gs://'] if self.sub_opts: for o, a in self.sub_opts: if o == '-p': self.project_id = a if not self.project_id: self.project_id = PopulateProjectId(None) # Request the service account for that project; this might create the # service account if it doesn't already exist. self.logger.debug('Checking service account for project %s', self.project_id) service_account = self.gsutil_api.GetProjectServiceAccount( self.project_id, provider='gs').email_address print(service_account) return 0 def _RunSubCommand(self, func): try: self.sub_opts, self.args = getopt.getopt( self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API. If you ' 'have only supplied hmac credentials in your boto file, please ' 'instead supply a credential type that can be used with the JSON ' 'API.' % self.command_name))) def RunCommand(self): """Command entry point for the kms command.""" # If the only credential type the user supplies in their boto file is hmac, # GetApiSelector logic will force us to use the XML API. As the XML API does # not support all the operations needed for kms subcommands, fail early. if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the GCS JSON API, which ' 'cannot use HMAC credentials. Please supply a credential ' 'type that is compatible with the JSON API (e.g. OAuth2) in your ' 'boto config file.' % self.command_name))) method_for_subcommand = { 'authorize': KmsCommand._Authorize, 'encryption': KmsCommand._Encryption, 'serviceaccount': KmsCommand._ServiceAccount } self.subcommand_name = self.args.pop(0) if self.subcommand_name in method_for_subcommand: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand(method_for_subcommand[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name))	20.74 KB	2025-06-25 08:19
📄 label.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of label command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import codecs import json import os import six from gslib import metrics from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import shim_util from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LabelTranslation _SET_SYNOPSIS = """ gsutil label set <label-json-file> gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil label get gs://<bucket_name> """ _CH_SYNOPSIS = """ gsutil label ch <label_modifier>... gs://<bucket_name>... where each <label_modifier> is one of the following forms: -l <key>:<value> -d <key> """ _GET_DESCRIPTION = """ <B>GET</B> The "label get" command gets the `labels <https://cloud.google.com/storage/docs/tags-and-labels#bucket-labels>`_ applied to a bucket, which you can save and edit for use with the "label set" command. """ _SET_DESCRIPTION = """ <B>SET</B> The "label set" command allows you to set the labels on one or more buckets. You can retrieve a bucket's labels using the "label get" command, save the output to a file, edit the file, and then use the "label set" command to apply those labels to the specified bucket(s). For example: gsutil label get gs://bucket > labels.json Make changes to labels.json, such as adding an additional label, then: gsutil label set labels.json gs://example-bucket Note that you can set these labels on multiple buckets at once: gsutil label set labels.json gs://bucket-foo gs://bucket-bar """ _CH_DESCRIPTION = """ <B>CH</B> The "label ch" command updates a bucket's label configuration, applying the label changes specified by the -l and -d flags. You can specify multiple label changes in a single command run; all changes will be made atomically to each bucket. <B>CH EXAMPLES</B> Examples for "ch" sub-command: Add the label "key-foo:value-bar" to the bucket "example-bucket": gsutil label ch -l key-foo:value-bar gs://example-bucket Change the above label to have a new value: gsutil label ch -l key-foo:other-value gs://example-bucket Add a new label and delete the old one from above: gsutil label ch -l new-key:new-value -d key-foo gs://example-bucket <B>CH OPTIONS</B> The "ch" sub-command has the following options -l Add or update a label with the specified key and value. -d Remove the label with the specified key. """ _SYNOPSIS = (_SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + _CH_SYNOPSIS.lstrip('\n') + '\n\n') _DESCRIPTION = """ Gets, sets, or changes the label configuration (also called the tagging configuration by other storage providers) of one or more buckets. An example label JSON document looks like the following: { "your_label_key": "your_label_value", "your_other_label_key": "your_other_label_value" } The label command has three sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + _CH_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _ch_help_text = CreateHelpText(_CH_SYNOPSIS, _CH_DESCRIPTION) class LabelCommand(Command): """Implementation of gsutil label command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'label', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='l:d:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'ch': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='label', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Get, set, or change the label configuration of a bucket.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'ch': _ch_help_text, }, ) gcloud_storage_map = GcloudStorageMap(gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=labels,empty=\' has no label ' 'configuration.\',empty_prefix_key=storage_url,indent=2]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update', '--labels-file'], flag_map={}, ), 'ch': GcloudStorageMap( gcloud_command=['storage', 'buckets', 'update'], flag_map={ '-d': GcloudStorageFlag( '--remove-labels', repeat_type=shim_util.RepeatFlagType.LIST), '-l': GcloudStorageFlag( '--update-labels', repeat_type=shim_util.RepeatFlagType.DICT), }, ), }, flag_map={}) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 # Filename comes before bucket arg(s). return 1 def _SetLabel(self): """Parses options and sets labels on the specified buckets.""" # At this point, "set" has been popped off the front of self.args. if len(self.args) < 2: self.RaiseWrongNumberOfArgumentsException() label_filename = self.args[0] if not os.path.isfile(label_filename): raise CommandException('Could not find the file "%s".' % label_filename) with codecs.open(label_filename, 'r', UTF8) as label_file: label_text = label_file.read() @Retry(PreconditionException, tries=3, timeout_secs=1) def _SetLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) if url.scheme == 's3': # Uses only XML. self.gsutil_api.XmlPassThroughSetTagging(label_text, url, provider=url.scheme) else: # Must be a 'gs://' bucket. labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None new_label_json = json.loads(label_text) if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # Perform a read-modify-write so that we can specify which # existing labels need to be deleted. _, bucket_metadata = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Set all old keys' values to None; this will delete each key that # is not included in the new set of labels. merged_labels = dict( (key, None) for key, _ in six.iteritems(label_json)) merged_labels.update(new_label_json) labels_message = LabelTranslation.DictToMessage(merged_labels) else: # ApiSelector.XML # No need to read-modify-write with the XML API. labels_message = LabelTranslation.DictToMessage(new_label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args[1:] for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for bucket_listing_ref in bucket_iter: some_matched = True _SetLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _ChLabel(self): """Parses options and changes labels on the specified buckets.""" self.label_changes = {} self.num_deletions = 0 if self.sub_opts: for o, a in self.sub_opts: if o == '-l': label_split = a.split(':') if len(label_split) != 2: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To add a label, please use the form <key>:<value>.' % a) self.label_changes[label_split[0]] = label_split[1] elif o == '-d': # Ensure only the key is supplied; stop if key:value was given. val_split = a.split(':') if len(val_split) != 1: raise CommandException( 'Found incorrectly formatted option for "gsutil label ch": ' '"%s". To delete a label, provide only its key.' % a) self.label_changes[a] = None self.num_deletions += 1 else: self.RaiseInvalidArgumentException() if not self.label_changes: raise CommandException( 'Please specify at least one label change with the -l or -d flags.') @Retry(PreconditionException, tries=3, timeout_secs=1) def _ChLabelForBucket(blr): url = blr.storage_url self.logger.info('Setting label configuration on %s...', blr) labels_message = None # When performing a read-modify-write cycle, include metageneration to # avoid race conditions (supported for GS buckets only). metageneration = None if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.JSON): # The JSON API's PATCH semantics allow us to skip read-modify-write, # with the exception of one edge case - attempting to delete a # nonexistent label returns an error iff no labels previously existed corrected_changes = self.label_changes if self.num_deletions: (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) if not bucket_metadata.labels: metageneration = bucket_metadata.metageneration # Remove each change that would try to delete a nonexistent key. corrected_changes = dict( (k, v) for k, v in six.iteritems(self.label_changes) if v) labels_message = LabelTranslation.DictToMessage(corrected_changes) else: # ApiSelector.XML # Perform a read-modify-write cycle so that we can specify which # existing labels need to be deleted. (_, bucket_metadata) = self.GetSingleBucketUrlFromArg( url.url_string, bucket_fields=['labels', 'metageneration']) metageneration = bucket_metadata.metageneration label_json = {} if bucket_metadata.labels: label_json = json.loads( LabelTranslation.JsonFromMessage(bucket_metadata.labels)) # Modify label_json such that all specified labels are added # (overwriting old labels if necessary) and all specified deletions # are removed from label_json if already present. for key, value in six.iteritems(self.label_changes): if not value and key in label_json: del label_json[key] else: label_json[key] = value labels_message = LabelTranslation.DictToMessage(label_json) preconditions = Preconditions(meta_gen_match=metageneration) bucket_metadata = apitools_messages.Bucket(labels=labels_message) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, preconditions=preconditions, provider=url.scheme, fields=['id']) some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True _ChLabelForBucket(bucket_listing_ref) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetAndPrintLabel(self, bucket_arg): """Gets and prints the labels for a cloud bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( bucket_arg, bucket_fields=['labels']) if bucket_url.scheme == 's3': print((self.gsutil_api.XmlPassThroughGetTagging( bucket_url, provider=bucket_url.scheme))) else: if bucket_metadata.labels: print((LabelTranslation.JsonFromMessage(bucket_metadata.labels, pretty_print=True))) else: print(('%s has no label configuration.' % bucket_url)) def RunCommand(self): """Command entry point for the label command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) if action_subcommand == 'get': metrics.LogCommandParams(subcommands=[action_subcommand]) self._GetAndPrintLabel(self.args[0]) elif action_subcommand == 'set': metrics.LogCommandParams(subcommands=[action_subcommand]) self._SetLabel() elif action_subcommand == 'ch': metrics.LogCommandParams(subcommands=[action_subcommand]) self._ChLabel() else: raise CommandException( 'Invalid subcommand "%s" for the %s command.\nSee "gsutil help %s".' % (action_subcommand, self.command_name, self.command_name)) return 0	15.84 KB	2025-06-25 08:19
📄 lifecycle.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of lifecycle configuration command for GCS buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import LifecycleTranslation _GET_SYNOPSIS = """ gsutil lifecycle get gs://<bucket_name> """ _SET_SYNOPSIS = """ gsutil lifecycle set <config-json-file> gs://<bucket_name>... """ _SYNOPSIS = _GET_SYNOPSIS + _SET_SYNOPSIS.lstrip('\n') + '\n' _GET_DESCRIPTION = """ <B>GET</B> Gets the lifecycle management configuration for a given bucket. You can get the lifecycle management configuration for only one bucket at a time. To update the configuration, you can redirect the output of the ``get`` command into a file, edit the file, and then set it on the bucket using the ``set`` sub-command. """ _SET_DESCRIPTION = """ <B>SET</B> Sets the lifecycle management configuration on one or more buckets. The ``config-json-file`` specified on the command line should be a path to a local file containing the lifecycle configuration JSON document. """ _DESCRIPTION = """ You can use the ``lifecycle`` command to get or set lifecycle management policies for a given bucket. This command is supported for buckets only, not objects. For more information, see `Object Lifecycle Management <https://cloud.google.com/storage/docs/lifecycle>`_. The ``lifecycle`` command has two sub-commands: """ + _GET_DESCRIPTION + _SET_DESCRIPTION + """ <B>EXAMPLES</B> The following lifecycle management configuration JSON document specifies that all objects in this bucket that are more than 365 days old are deleted automatically: { "rule": [ { "action": {"type": "Delete"}, "condition": {"age": 365} } ] } The following empty lifecycle management configuration JSON document removes all lifecycle configuration for a bucket: {} """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LifecycleCommand(Command): """Implementation of gsutil lifecycle command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'lifecycle', command_name_aliases=['lifecycleconfig'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=True, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.JSON, ApiSelector.XML, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument.MakeNFileURLsArgument(1), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1),] }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='lifecycle', help_name_aliases=[ 'getlifecycle', 'setlifecycle', ], help_type='command_help', help_one_line_summary=('Get or set lifecycle configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lifecycle-file={}'.format(self.args[1]), ] + self.args[2:], # Adds target bucket URLs. flag_map={}, ), }, flag_map={}, ) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=lifecycle_config,empty=\' ' 'has no lifecycle configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _SetLifecycleConfig(self): """Sets lifecycle configuration for a Google Cloud Storage bucket.""" lifecycle_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider 'lifecycle set' requests. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Open, read and parse file containing JSON document. lifecycle_file = open(lifecycle_arg, 'r') lifecycle_txt = lifecycle_file.read() lifecycle_file.close() # Iterate over URLs, expanding wildcards and setting the lifecycle on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['lifecycle']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting lifecycle configuration on %s...', blr) if url.scheme == 's3': self.gsutil_api.XmlPassThroughSetLifecycle(lifecycle_txt, url, provider=url.scheme) else: lifecycle = LifecycleTranslation.JsonLifecycleToMessage(lifecycle_txt) bucket_metadata = apitools_messages.Bucket(lifecycle=lifecycle) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _GetLifecycleConfig(self): """Gets lifecycle configuration for a Google Cloud Storage bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['lifecycle']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetLifecycle( bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.lifecycle and bucket_metadata.lifecycle.rule: sys.stdout.write( LifecycleTranslation.JsonLifecycleFromMessage( bucket_metadata.lifecycle)) else: sys.stdout.write('%s has no lifecycle configuration.\n' % bucket_url) return 0 def RunCommand(self): """Command entry point for the lifecycle command.""" subcommand = self.args.pop(0) if subcommand == 'get': metrics.LogCommandParams(subcommands=[subcommand]) return self._GetLifecycleConfig() elif subcommand == 'set': metrics.LogCommandParams(subcommands=[subcommand]) return self._SetLifecycleConfig() else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (subcommand, self.command_name))	8.71 KB	2025-06-25 08:19
📄 logging.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of logging configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import text_util _SET_SYNOPSIS = """ gsutil logging set on -b <logging_bucket_name> [-o <log_object_prefix>] gs://<bucket_name>... gsutil logging set off gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil logging get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') + '\n' _SET_DESCRIPTION = """ <B>SET</B> The ``set`` sub-command has two sub-commands: <B>ON</B> The ``gsutil logging set on`` command enables usage and storage logging for the buckets named by the specified URLs, outputting log files to the bucket specified with the ``-b`` flag. Cloud Storage doesn't validate the existence of the output bucket, so users should ensure it already exists, and all URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). The optional ``-o`` flag specifies the prefix for log object names. The default prefix is the bucket name. For example, the command: gsutil logging set on -b gs://my_logging_bucket -o UsageLog \\ gs://my_bucket1 gs://my_bucket2 causes all read and write activity to objects in ``gs://mybucket1`` and ``gs://mybucket2`` to be logged to objects prefixed with the name ``UsageLog``, with those log objects written to the bucket ``gs://my_logging_bucket``. In addition to enabling logging on your bucket(s), you also need to grant cloud-storage-analytics@google.com write access to the log bucket, using this command: gsutil acl ch -g cloud-storage-analytics@google.com:W gs://my_logging_bucket Note that log data may contain sensitive information, so you should make sure to set an appropriate default bucket ACL to protect that data. (See "gsutil help defacl".) <B>OFF</B> This command disables usage and storage logging for the buckets named by the specified URLs. All URLs must name Cloud Storage buckets (e.g., ``gs://bucket``). No logging data is removed from the log buckets when you disable logging, but Google Cloud Storage stops delivering new logs once you have run this command. """ _GET_DESCRIPTION = """ <B>GET</B> If logging is enabled for the specified bucket url, the server responds with a JSON document that looks something like this: { "logBucket": "my_logging_bucket", "logObjectPrefix": "UsageLog" } You can download log data from your log bucket using the gsutil cp command. """ _DESCRIPTION = """ Google Cloud Storage offers `usage logs and storage logs <https://cloud.google.com/storage/docs/access-logs>`_ in the form of CSV files that you can download and view. Usage logs provide information for all of the requests made on a specified bucket and are created hourly. Storage logs provide information about the storage consumption of that bucket for the last day and are created daily. Once set up, usage logs and storage logs are automatically created as new objects in a bucket that you specify. Usage logs and storage logs are subject to the same pricing as other objects stored in Cloud Storage. For a complete list of usage log fields and storage data fields, see `Usage and storage log format <https://cloud.google.com/storage/docs/access-logs#format>`_. The ``logging`` command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION + """ <B>OPTIONS</B> -b bucket_name Specifies the bucket that stores the generated logs. This flag is only available for the ``set on`` command and is required for that command. -o log_prefix Specifies a common prefix for the names of generated logs. This flag is only available for the ``set on`` command and is optional for that command. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class LoggingCommand(Command): """Implementation of gsutil logging command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'logging', command_name_aliases=['disablelogging', 'enablelogging', 'getlogging'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='b:o:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='logging', help_name_aliases=[ 'loggingconfig', 'logs', 'log', 'getlogging', 'enablelogging', 'disablelogging', ], help_type='command_help', help_one_line_summary='Configure or retrieve logging on buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=logging_config,empty=\' has ' 'no logging configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-b': GcloudStorageFlag('--log-bucket'), '-o': GcloudStorageFlag('--log-object-prefix'), }, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-log-bucket', '--clear-log-object-prefix', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _Get(self): """Gets logging configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['logging']) if bucket_url.scheme == 's3': text_util.print_to_fd(self.gsutil_api.XmlPassThroughGetLogging( bucket_url, provider=bucket_url.scheme), end='') else: if (bucket_metadata.logging and bucket_metadata.logging.logBucket and bucket_metadata.logging.logObjectPrefix): text_util.print_to_fd( str(encoding.MessageToJson(bucket_metadata.logging))) else: text_util.print_to_fd('%s has no logging configuration.' % bucket_url) return 0 def _Enable(self): """Enables logging configuration for a bucket.""" # Disallow multi-provider 'logging set on' calls, because the schemas # differ. if not UrlsAreForSingleProvider(self.args): raise CommandException('"logging set on" command spanning providers not ' 'allowed.') target_bucket_url = None target_prefix = None for opt, opt_arg in self.sub_opts: if opt == '-b': target_bucket_url = StorageUrlFromString(opt_arg) if opt == '-o': target_prefix = opt_arg if not target_bucket_url: raise CommandException('"logging set on" requires \'-b <log_bucket>\' ' 'option') if not target_bucket_url.IsBucket(): raise CommandException('-b option must specify a bucket URL.') # Iterate over URLs, expanding wildcards and setting logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Enabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue( logBucket=target_bucket_url.bucket_name, logObjectPrefix=target_prefix or url.bucket_name) bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def _Disable(self): """Disables logging configuration for a bucket.""" # Iterate over URLs, expanding wildcards, and disabling logging on each. some_matched = False for url_str in self.args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Disabling logging on %s...', blr) logging = apitools_messages.Bucket.LoggingValue() bucket_metadata = apitools_messages.Bucket(logging=logging) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 0 def RunCommand(self): """Command entry point for the logging command.""" # Parse the subcommand and alias for the new logging command. action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._Get metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': state_subcommand = self.args.pop(0) if not self.args: self.RaiseWrongNumberOfArgumentsException() if state_subcommand == 'on': func = self._Enable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) elif state_subcommand == 'off': func = self._Disable metrics.LogCommandParams( subcommands=[action_subcommand, state_subcommand]) else: raise CommandException( ('Invalid subcommand "%s" for the "%s %s" command.\n' 'See "gsutil help logging".') % (state_subcommand, self.command_name, action_subcommand)) else: raise CommandException(('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help logging".') % (action_subcommand, self.command_name)) self.ParseSubOpts(check_args=True) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) func() return 0	12.93 KB	2025-06-25 08:19
📄 ls.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like ls command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import six from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import LsHelper from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils import text_util from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import LabelTranslation from gslib.utils.unit_util import MakeHumanReadable # Regex that assists with converting JSON timestamp to ls-style output. # This excludes timestamp fractional seconds, for example: # 2013-07-03 20:32:53.048000+00:00 JSON_TIMESTAMP_RE = re.compile(r'([^\s])\s([^\.\+]).') _SYNOPSIS = """ gsutil ls [-a] [-b] [-d] [-l] [-L] [-r] [-p proj_id] url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Retrieves a list of providers, buckets, or objects matching the criteria, ordered in the list lexicographically by name. <B>LISTING PROVIDERS, BUCKETS, SUBDIRECTORIES, AND OBJECTS</B> If you run ``gsutil ls`` without URLs, it lists all of the Google Cloud Storage buckets under your default project ID (or all of the Cloud Storage buckets under the project you specify with the ``-p`` flag): gsutil ls If you specify one or more provider URLs, ``gsutil ls`` lists buckets at each listed provider: gsutil ls gs:// gsutil currently supports ``gs://`` and ``s3://`` as valid providers If you specify bucket URLs, or use `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_ to capture a set of buckets, ``gsutil ls`` lists objects at the top level of each bucket, along with the names of each subdirectory. For example: gsutil ls gs://bucket might produce output like: gs://bucket/obj1.htm gs://bucket/obj2.htm gs://bucket/images1/ gs://bucket/images2/ The "/" at the end of the last 2 URLs tells you they are subdirectories, which you can list using: gsutil ls gs://bucket/images If you specify object URLs, ``gsutil ls`` lists the specified objects. For example: gsutil ls gs://bucket/.txt lists all files whose name matches the above wildcard at the top level of the bucket. For more details, see `URI wildcards <https://cloud.google.com/storage/docs/wildcards>`_. <B>DIRECTORY BY DIRECTORY, FLAT, and RECURSIVE LISTINGS</B> Listing a bucket or subdirectory (as illustrated near the end of the previous section) only shows the objects and names of subdirectories it contains. You can list all objects in a bucket by using the -r option. For example: gsutil ls -r gs://bucket lists the top-level objects and buckets, then the objects and buckets under gs://bucket/images1, then those under gs://bucket/images2, etc. If you want to see all objects in the bucket in one "flat" listing use the recursive ("") wildcard, like: gsutil ls -r gs://bucket/* or, for a flat listing of a subdirectory: gsutil ls -r gs://bucket/dir/** If you want to see only the subdirectory itself, use the -d option: gsutil ls -d gs://bucket/dir <B>LISTING OBJECT DETAILS</B> If you specify the -l option, gsutil outputs additional information about each matching provider, bucket, subdirectory, or object. For example: gsutil ls -l gs://bucket/.html gs://bucket/.txt prints the object size, creation time stamp, and name of each matching object, along with the total count and sum of sizes of all matching objects: 2276224 2020-03-02T19:25:17Z gs://bucket/obj1.html 3914624 2020-03-02T19:30:27Z gs://bucket/obj2.html 131 2020-03-02T19:37:45Z gs://bucket/obj3.txt TOTAL: 3 objects, 6190979 bytes (5.9 MiB) Note that the total listed in parentheses above is in mebibytes (or gibibytes, tebibytes, etc.), which corresponds to the unit of billing measurement for Google Cloud Storage. You can get a listing of all the objects in the top-level bucket directory (along with the total count and sum of sizes) using a command like: gsutil ls -l gs://bucket To print additional detail about objects and buckets use the gsutil ls -L option. For example: gsutil ls -L gs://bucket/obj1 prints something like: gs://bucket/obj1: Creation time: Fri, 26 May 2017 22:55:44 GMT Update time: Tue, 18 Jul 2017 12:31:18 GMT Storage class: STANDARD Content-Length: 60183 Content-Type: image/jpeg Hash (crc32c): zlUhtg== Hash (md5): Bv86IAzFzrD1Z2io/c7yqA== ETag: 5ca67960a586723b7344afffc81 Generation: 1378862725952000 Metageneration: 1 ACL: [ { "entity": "project-owners-867484910061", "projectTeam": { "projectNumber": "867484910061", "team": "owners" }, "role": "OWNER" }, { "email": "jane@gmail.com", "entity": "user-jane@gmail.com", "role": "OWNER" } ] TOTAL: 1 objects, 60183 bytes (58.77 KiB) Note that results may contain additional fields, such as custom metadata or a storage class update time, if they are applicable to the object. Also note that some fields, such as update time, are not available with the (non-default) XML API. See also "gsutil help acl" for getting a more readable version of the ACL. <B>LISTING BUCKET DETAILS</B> If you want to see information about the bucket itself, use the -b option. For example: gsutil ls -L -b gs://bucket prints something like: gs://bucket/ : Storage class: STANDARD Location constraint: US Versioning enabled: False Logging configuration: None Website configuration: None CORS configuration: None Lifecycle configuration: None Requester Pays enabled: True Labels: None Default KMS key: None Time created: Thu, 14 Jan 2016 19:25:17 GMT Time updated: Thu, 08 Jun 2017 21:17:59 GMT Metageneration: 1 Bucket Policy Only enabled: False ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Default ACL: [ { "entity": "project-owners-867489160491", "projectTeam": { "projectNumber": "867489160491", "team": "owners" }, "role": "OWNER" } ] Note that some fields above (time created, time updated, metageneration) are not available with the (non-default) XML API. <B>OPTIONS</B> -l Prints long listing (owner, length). -L Prints even more detail than -l. Note: If you use this option with the (non-default) XML API it generates an additional request per object being listed, which makes the -L option run much more slowly and cost more than the default JSON API. -d List matching subdirectory names instead of contents, and do not recurse into matching subdirectories even if the -R option is specified. -b Prints info about the bucket when used with a bucket URL. -h When used with -l, prints object sizes in human readable format (e.g., 1 KiB, 234 MiB, 2 GiB, etc.) -p proj_id Specifies the project ID or project number to use for listing buckets. -R, -r Requests a recursive listing, performing at least one listing operation per subdirectory. If you have a large number of subdirectories and do not require recursive-style output ordering, you may be able to instead use wildcards to perform a flat listing, e.g. ``gsutil ls gs://mybucket/``, which generally performs fewer listing operations. -a Includes non-current object versions / generations in the listing (only useful with a versioning-enabled bucket). If combined with -l option also prints metageneration for each listed object. -e Include ETag in long listing (-l) output. """) class ListingStyle(object): """Enum class for specifying listing style.""" SHORT = 'SHORT' LONG = 'LONG' LONG_LONG = 'LONG_LONG' class LsCommand(Command): """Implementation of gsutil ls command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'ls', command_name_aliases=[ 'dir', 'list', ], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='aebdlLhp:rR', file_url_ok=False, provider_url_ok=True, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ls', help_name_aliases=[ 'dir', 'list', ], help_type='command_help', help_one_line_summary='List providers, buckets, or objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # TODO(b/206151616) Add mappings for remaining flags. gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'ls', '--fetch-encrypted-object-hashes'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-l': GcloudStorageFlag('-l'), '-L': GcloudStorageFlag('-L'), '-b': GcloudStorageFlag('-b'), '-e': GcloudStorageFlag('-e'), '-a': GcloudStorageFlag('-a'), '-h': GcloudStorageFlag('--readable-sizes'), '-p': GcloudStorageFlag('--project'), }, ) def _PrintBucketInfo(self, bucket_blr, listing_style): """Print listing info for given bucket. Args: bucket_blr: BucketListingReference for the bucket being listed listing_style: ListingStyle enum describing type of output desired. Returns: Tuple (total objects, total bytes) in the bucket. """ if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): text_util.print_to_fd(bucket_blr) return # listing_style == ListingStyle.LONG_LONG: # We're guaranteed by the caller that the root object is populated. bucket = bucket_blr.root_object location_constraint = bucket.location storage_class = bucket.storageClass fields = { 'bucket': bucket_blr.url_string, 'storage_class': storage_class, 'location_constraint': location_constraint, 'acl': AclTranslation.JsonFromMessage(bucket.acl), 'default_acl': AclTranslation.JsonFromMessage(bucket.defaultObjectAcl), 'versioning': bucket.versioning and bucket.versioning.enabled, 'website_config': 'Present' if bucket.website else 'None', 'logging_config': 'Present' if bucket.logging else 'None', 'cors_config': 'Present' if bucket.cors else 'None', 'lifecycle_config': 'Present' if bucket.lifecycle else 'None', 'requester_pays': bucket.billing and bucket.billing.requesterPays } if bucket.retentionPolicy: fields['retention_policy'] = 'Present' if bucket.labels: fields['labels'] = LabelTranslation.JsonFromMessage(bucket.labels, pretty_print=True) else: fields['labels'] = 'None' if bucket.encryption and bucket.encryption.defaultKmsKeyName: fields['default_kms_key'] = bucket.encryption.defaultKmsKeyName else: fields['default_kms_key'] = 'None' fields['encryption_config'] = 'Present' if bucket.encryption else 'None' # Fields not available in all APIs (e.g. the XML API) if bucket.autoclass and bucket.autoclass.enabled: fields['autoclass_enabled_date'] = ( bucket.autoclass.toggleTime.strftime('%a, %d %b %Y')) if bucket.locationType: fields['location_type'] = bucket.locationType if bucket.customPlacementConfig: fields['custom_placement_locations'] = ( bucket.customPlacementConfig.dataLocations) if bucket.metageneration: fields['metageneration'] = bucket.metageneration if bucket.timeCreated: fields['time_created'] = bucket.timeCreated.strftime( '%a, %d %b %Y %H:%M:%S GMT') if bucket.updated: fields['updated'] = bucket.updated.strftime('%a, %d %b %Y %H:%M:%S GMT') if bucket.defaultEventBasedHold: fields['default_eventbased_hold'] = bucket.defaultEventBasedHold if bucket.iamConfiguration: if bucket.iamConfiguration.bucketPolicyOnly: enabled = bucket.iamConfiguration.bucketPolicyOnly.enabled fields['bucket_policy_only_enabled'] = enabled if bucket.iamConfiguration.publicAccessPrevention: fields[ 'public_access_prevention'] = bucket.iamConfiguration.publicAccessPrevention if bucket.rpo: fields['rpo'] = bucket.rpo if bucket.satisfiesPZS: fields['satisfies_pzs'] = bucket.satisfiesPZS # For field values that are multiline, add indenting to make it look # prettier. for key in fields: previous_value = fields[key] if (not isinstance(previous_value, six.string_types) or '\n' not in previous_value): continue new_value = previous_value.replace('\n', '\n\t ') # Start multiline values on a new line if they aren't already. if not new_value.startswith('\n'): new_value = '\n\t ' + new_value fields[key] = new_value # Only display certain properties if the given API returned them (JSON API # returns many fields that the XML API does not). autoclass_line = '' location_type_line = '' custom_placement_locations_line = '' metageneration_line = '' time_created_line = '' time_updated_line = '' default_eventbased_hold_line = '' retention_policy_line = '' bucket_policy_only_enabled_line = '' public_access_prevention_line = '' rpo_line = '' satisifies_pzs_line = '' if 'autoclass_enabled_date' in fields: autoclass_line = '\tAutoclass:\t\t\tEnabled on {autoclass_enabled_date}\n' if 'location_type' in fields: location_type_line = '\tLocation type:\t\t\t{location_type}\n' if 'custom_placement_locations' in fields: custom_placement_locations_line = ( '\tPlacement locations:\t\t{custom_placement_locations}\n') if 'metageneration' in fields: metageneration_line = '\tMetageneration:\t\t\t{metageneration}\n' if 'time_created' in fields: time_created_line = '\tTime created:\t\t\t{time_created}\n' if 'updated' in fields: time_updated_line = '\tTime updated:\t\t\t{updated}\n' if 'default_eventbased_hold' in fields: default_eventbased_hold_line = ( '\tDefault Event-Based Hold:\t{default_eventbased_hold}\n') if 'retention_policy' in fields: retention_policy_line = '\tRetention Policy:\t\t{retention_policy}\n' if 'bucket_policy_only_enabled' in fields: bucket_policy_only_enabled_line = ('\tBucket Policy Only enabled:\t' '{bucket_policy_only_enabled}\n') if 'public_access_prevention' in fields: public_access_prevention_line = ('\tPublic access prevention:\t' '{public_access_prevention}\n') if 'rpo' in fields: rpo_line = ('\tRPO:\t\t\t\t{rpo}\n') if 'satisfies_pzs' in fields: satisifies_pzs_line = '\tSatisfies PZS:\t\t\t{satisfies_pzs}\n' text_util.print_to_fd( ('{bucket} :\n' '\tStorage class:\t\t\t{storage_class}\n' + location_type_line + '\tLocation constraint:\t\t{location_constraint}\n' + custom_placement_locations_line + '\tVersioning enabled:\t\t{versioning}\n' '\tLogging configuration:\t\t{logging_config}\n' '\tWebsite configuration:\t\t{website_config}\n' '\tCORS configuration: \t\t{cors_config}\n' '\tLifecycle configuration:\t{lifecycle_config}\n' '\tRequester Pays enabled:\t\t{requester_pays}\n' + retention_policy_line + default_eventbased_hold_line + '\tLabels:\t\t\t\t{labels}\n' + '\tDefault KMS key:\t\t{default_kms_key}\n' + time_created_line + time_updated_line + metageneration_line + bucket_policy_only_enabled_line + autoclass_line + public_access_prevention_line + rpo_line + satisifies_pzs_line + '\tACL:\t\t\t\t{acl}\n' '\tDefault ACL:\t\t\t{default_acl}').format(fields)) if bucket_blr.storage_url.scheme == 's3': text_util.print_to_fd( 'Note: this is an S3 bucket so configuration values may be ' 'blank. To retrieve bucket configuration values, use ' 'individual configuration commands such as gsutil acl get ' '<bucket>.') def _PrintLongListing(self, bucket_listing_ref): """Prints an object with ListingStyle.LONG.""" obj = bucket_listing_ref.root_object url_str = bucket_listing_ref.url_string if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): size_string = '0' num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: size_string = (MakeHumanReadable(obj.size) if self.human_readable else str(obj.size)) num_bytes = obj.size num_objs = 1 timestamp = JSON_TIMESTAMP_RE.sub(r'\1T\2Z', str(obj.timeCreated)) printstr = '%(size)10s %(timestamp)s %(url)s' encoded_etag = None encoded_metagen = None if self.all_versions: printstr += ' metageneration=%(metageneration)s' encoded_metagen = str(obj.metageneration) if self.include_etag: printstr += ' etag=%(etag)s' encoded_etag = obj.etag format_args = { 'size': size_string, 'timestamp': timestamp, 'url': url_str, 'metageneration': encoded_metagen, 'etag': encoded_etag } text_util.print_to_fd(printstr % format_args) return (num_objs, num_bytes) def RunCommand(self): """Command entry point for the ls command.""" got_nomatch_errors = False got_bucket_nomatch_errors = False listing_style = ListingStyle.SHORT get_bucket_info = False self.recursion_requested = False self.all_versions = False self.include_etag = False self.human_readable = False self.list_subdir_contents = True if self.sub_opts: for o, a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-e': self.include_etag = True elif o == '-b': get_bucket_info = True elif o == '-h': self.human_readable = True elif o == '-l': listing_style = ListingStyle.LONG elif o == '-L': listing_style = ListingStyle.LONG_LONG elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-d': self.list_subdir_contents = False if not self.args: # default to listing all gs buckets self.args = ['gs://'] total_objs = 0 total_bytes = 0 def MaybePrintBucketHeader(blr): if len(self.args) > 1: text_util.print_to_fd('%s:' % six.ensure_text(blr.url_string)) print_bucket_header = MaybePrintBucketHeader for url_str in self.args: storage_url = StorageUrlFromString(url_str) if storage_url.IsFileUrl(): raise CommandException('Only cloud URLs are supported for %s' % self.command_name) bucket_fields = None if (listing_style == ListingStyle.SHORT or listing_style == ListingStyle.LONG): bucket_fields = ['id'] elif listing_style == ListingStyle.LONG_LONG: bucket_fields = [ 'acl', 'autoclass', 'billing', 'cors', 'customPlacementConfig', 'defaultObjectAcl', 'encryption', 'iamConfiguration', 'labels', 'location', 'locationType', 'logging', 'lifecycle', 'metageneration', 'retentionPolicy', 'defaultEventBasedHold', 'rpo', 'satisfiesPZS', 'storageClass', 'timeCreated', 'updated', 'versioning', 'website', ] if storage_url.IsProvider(): # Provider URL: use bucket wildcard to list buckets. for blr in self.WildcardIterator( '%s://*' % storage_url.scheme).IterBuckets(bucket_fields=bucket_fields): self._PrintBucketInfo(blr, listing_style) elif storage_url.IsBucket() and get_bucket_info: # ls -b bucket listing request: List info about bucket(s). total_buckets = 0 for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): if not ContainsWildcard(url_str) and not blr.root_object: # Iterator does not make an HTTP call for non-wildcarded # listings with fields=='id'. Ensure the bucket exists by calling # GetBucket. self.gsutil_api.GetBucket(blr.storage_url.bucket_name, fields=['id'], provider=storage_url.scheme) self._PrintBucketInfo(blr, listing_style) total_buckets += 1 if not ContainsWildcard(url_str) and not total_buckets: got_bucket_nomatch_errors = True else: # URL names a bucket, object, or object subdir -> # list matching object(s) / subdirs. def _PrintPrefixLong(blr): text_util.print_to_fd('%-33s%s' % ('', six.ensure_text(blr.url_string))) if listing_style == ListingStyle.SHORT: # ls helper by default readies us for a short listing. listing_helper = LsHelper( self.WildcardIterator, self.logger, all_versions=self.all_versions, print_bucket_header_func=print_bucket_header, should_recurse=self.recursion_requested, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG: bucket_listing_fields = [ 'name', 'size', 'timeCreated', 'updated', ] if self.all_versions: bucket_listing_fields.extend([ 'generation', 'metageneration', ]) if self.include_etag: bucket_listing_fields.append('etag') listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=self._PrintLongListing, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) elif listing_style == ListingStyle.LONG_LONG: # List all fields bucket_listing_fields = (UNENCRYPTED_FULL_LISTING_FIELDS + ENCRYPTED_FIELDS) listing_helper = LsHelper( self.WildcardIterator, self.logger, print_object_func=PrintFullInfoAboutObject, print_dir_func=_PrintPrefixLong, print_bucket_header_func=print_bucket_header, all_versions=self.all_versions, should_recurse=self.recursion_requested, fields=bucket_listing_fields, list_subdir_contents=self.list_subdir_contents) else: raise CommandException('Unknown listing style: %s' % listing_style) exp_dirs, exp_objs, exp_bytes = ( listing_helper.ExpandUrlAndPrint(storage_url)) if storage_url.IsObject() and exp_objs == 0 and exp_dirs == 0: got_nomatch_errors = True total_bytes += exp_bytes total_objs += exp_objs if total_objs and listing_style != ListingStyle.SHORT: text_util.print_to_fd( 'TOTAL: %d objects, %d bytes (%s)' % (total_objs, total_bytes, MakeHumanReadable(float(total_bytes)))) if got_nomatch_errors: raise CommandException('One or more URLs matched no objects.') if got_bucket_nomatch_errors: raise NotFoundException('One or more bucket URLs matched no buckets.') return 0	26.7 KB	2025-06-25 08:19
📄 mb.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of mb command for creating cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re import textwrap from gslib.cloud_api import AccessDeniedException, BadRequestException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.rpo import VALID_RPO_VALUES from gslib.commands.rpo import VALID_RPO_VALUES_STRING from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAscii from gslib.utils.text_util import InsistOnOrOff from gslib.utils.text_util import NormalizeStorageClass from gslib.utils.encryption_helper import ValidateCMEK _SYNOPSIS = """ gsutil mb [-b (on\|off)] [-c <class>] [-k <key>] [-l <location>] [-p <project>] [--autoclass] [--retention <time>] [--pap <setting>] [--placement <region1>,<region2>] [--rpo {}] gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Create one or more new buckets. Google Cloud Storage has a single namespace, so you are not allowed to create a bucket with a name already in use by another user. You can, however, carve out parts of the bucket name space corresponding to your company's domain name (see "gsutil help naming"). If you don't specify a project ID or project number using the -p option, the buckets are created using the default project ID specified in your `gsutil configuration file <https://cloud.google.com/storage/docs/boto-gsutil>`_. The -l option specifies the location for the buckets. Once a bucket is created in a given location, it cannot be moved to a different location. Instead, you need to create a new bucket, move the data over, and then delete the original bucket. <B>BUCKET STORAGE CLASSES</B> You can specify one of the `storage classes <https://cloud.google.com/storage/docs/storage-classes>`_ for a bucket with the -c option. Example: gsutil mb -c nearline gs://some-bucket See online documentation for `pricing <https://cloud.google.com/storage/pricing>`_ and `SLA <https://cloud.google.com/storage/sla>`_ details. If you don't specify a -c option, the bucket is created with the default storage class Standard Storage. <B>BUCKET LOCATIONS</B> You can specify one of the `available locations <https://cloud.google.com/storage/docs/locations>`_ for a bucket with the -l option. Examples: gsutil mb -l asia gs://some-bucket gsutil mb -c standard -l us-east1 gs://some-bucket If you don't specify a -l option, the bucket is created in the default location (US). <B>Retention Policy</B> You can specify retention period in one of the following formats: --retention <number>s Specifies retention period of <number> seconds for objects in this bucket. --retention <number>d Specifies retention period of <number> days for objects in this bucket. --retention <number>m Specifies retention period of <number> months for objects in this bucket. --retention <number>y Specifies retention period of <number> years for objects in this bucket. Examples: gsutil mb --retention 1y gs://some-bucket gsutil mb --retention 36m gs://some-bucket If you don't specify a --retention option, the bucket is created with no retention policy. <B>OPTIONS</B> --autoclass Enables the Autoclass feature that automatically sets object storage classes. -b <on\|off> Specifies the uniform bucket-level access setting. When "on", ACLs assigned to objects in the bucket are not evaluated. Consequently, only IAM policies grant access to objects in these buckets. Default is "off". -c class Specifies the default storage class. Default is ``Standard``. See `Available storage classes <https://cloud.google.com/storage/docs/storage-classes#classes>`_ for a list of possible values. -k <key> Set the default KMS key using the full path to the key, which has the following form: ``projects/[project-id]/locations/[location]/keyRings/[key-ring]/cryptoKeys/[my-key]`` -l location Can be any supported location. See https://cloud.google.com/storage/docs/locations for a discussion of this distinction. Default is US. Locations are case insensitive. -p project Specifies the project ID or project number to create the bucket under. -s class Same as -c. --retention time Specifies the retention policy. Default is no retention policy. This can only be set on gs:// buckets and requires using the JSON API. For more details about retention policy see "gsutil help retention" --pap setting Specifies the public access prevention setting. Valid values are "enforced" or "inherited". When "enforced", objects in this bucket cannot be made publicly accessible. Default is "inherited". --placement reg1,reg2 Two regions that form the custom dual-region. Only regions within the same continent are or will ever be valid. Invalid location pairs (such as mixed-continent, or with unsupported regions) will return an error. --rpo setting Specifies the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_. This flag is not valid for single-region buckets, and multi-region buckets only accept a value of DEFAULT. Valid values for dual region buckets are {rpo_values}. If unspecified, DEFAULT is applied for dual-region and multi-region buckets. """.format(rpo_values=VALID_RPO_VALUES_STRING)) # Regex to disallow buckets violating charset or not [3..255] chars total. BUCKET_NAME_RE = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9\._-]{1,253}[a-zA-Z0-9]$') # Regex to disallow buckets with individual DNS labels longer than 63. TOO_LONG_DNS_NAME_COMP = re.compile(r'[-_a-z0-9]{64}') _RETENTION_FLAG = '--retention' IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue BucketPolicyOnlyValue = IamConfigurationValue.BucketPolicyOnlyValue class MbCommand(Command): """Implementation of gsutil mb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mb', command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='b:c:l:p:s:k:', supported_private_args=[ 'autoclass', 'retention=', 'pap=', 'placement=', 'rpo=' ], file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mb', help_name_aliases=[ 'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra', 'dras', 'reduced_availability', 'durable_reduced_availability', 'rr', 'reduced_redundancy', 'standard', 'storage class', 'nearline', 'nl', ], help_type='command_help', help_one_line_summary='Make buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'create'], flag_map={ '--autoclass': GcloudStorageFlag('--enable-autoclass'), '-b': GcloudStorageFlag({ 'on': '--uniform-bucket-level-access', 'off': None, }), '-c': GcloudStorageFlag('--default-storage-class'), '-s': GcloudStorageFlag('--default-storage-class'), '-k': GcloudStorageFlag('--default-encryption-key'), '-l': GcloudStorageFlag('--location'), '-p': GcloudStorageFlag('--project'), '--pap': GcloudStorageFlag({ 'enforced': '--public-access-prevention', 'inherited': None, }), '--placement': GcloudStorageFlag('--placement'), _RETENTION_FLAG: GcloudStorageFlag('--retention-period'), '--rpo': GcloudStorageFlag('--recovery-point-objective') }, ) def get_gcloud_storage_args(self): retention_arg_idx = 0 while retention_arg_idx < len(self.sub_opts): if self.sub_opts[retention_arg_idx][0] == _RETENTION_FLAG: break retention_arg_idx += 1 if retention_arg_idx < len(self.sub_opts): # Convert retention time to seconds, which gcloud knows how to handle. self.sub_opts[retention_arg_idx] = ( _RETENTION_FLAG, str(RetentionInSeconds(self.sub_opts[retention_arg_idx][1])) + 's') return super().get_gcloud_storage_args(MbCommand.gcloud_storage_map) def RunCommand(self): """Command entry point for the mb command.""" autoclass = False bucket_policy_only = None kms_key = None location = None storage_class = None seconds = None placements = None public_access_prevention = None rpo = None json_only_flags_in_command = [] if self.sub_opts: for o, a in self.sub_opts: if o == '--autoclass': autoclass = True json_only_flags_in_command.append(o) elif o == '-k': kms_key = a ValidateCMEK(kms_key) json_only_flags_in_command.append(o) elif o == '-l': location = a elif o == '-p': # Project IDs are sent as header values when using gs and s3 XML APIs. InsistAscii(a, 'Invalid non-ASCII character found in project ID') self.project_id = a elif o == '-c' or o == '-s': storage_class = NormalizeStorageClass(a) elif o == _RETENTION_FLAG: seconds = RetentionInSeconds(a) elif o == '--rpo': rpo = a.strip() if rpo not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for --rpo. Must be one of: {},' ' provided: {}'.format(VALID_RPO_VALUES_STRING, a)) json_only_flags_in_command.append(o) elif o == '-b': InsistOnOrOff(a, 'Only on and off values allowed for -b option') bucket_policy_only = (a == 'on') json_only_flags_in_command.append(o) elif o == '--pap': public_access_prevention = a json_only_flags_in_command.append(o) elif o == '--placement': placements = a.split(',') if len(placements) != 2: raise CommandException( 'Please specify two regions separated by comma without space.' ' Specified: {}'.format(a)) json_only_flags_in_command.append(o) bucket_metadata = apitools_messages.Bucket(location=location, rpo=rpo, storageClass=storage_class) if autoclass: bucket_metadata.autoclass = apitools_messages.Bucket.AutoclassValue( enabled=autoclass) if bucket_policy_only or public_access_prevention: bucket_metadata.iamConfiguration = IamConfigurationValue() iam_config = bucket_metadata.iamConfiguration if bucket_policy_only: iam_config.bucketPolicyOnly = BucketPolicyOnlyValue() iam_config.bucketPolicyOnly.enabled = bucket_policy_only if public_access_prevention: iam_config.publicAccessPrevention = public_access_prevention if kms_key: encryption = apitools_messages.Bucket.EncryptionValue() encryption.defaultKmsKeyName = kms_key bucket_metadata.encryption = encryption if placements: placement_config = apitools_messages.Bucket.CustomPlacementConfigValue() placement_config.dataLocations = placements bucket_metadata.customPlacementConfig = placement_config for bucket_url_str in self.args: bucket_url = StorageUrlFromString(bucket_url_str) if seconds is not None: if bucket_url.scheme != 'gs': raise CommandException('Retention policy can only be specified for ' 'GCS buckets.') retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) bucket_metadata.retentionPolicy = retention_policy if json_only_flags_in_command and self.gsutil_api.GetApiSelector( bucket_url.scheme) != ApiSelector.JSON: raise CommandException('The {} option(s) can only be used for GCS' ' Buckets with the JSON API'.format( ', '.join(json_only_flags_in_command))) if not bucket_url.IsBucket(): raise CommandException('The mb command requires a URL that specifies a ' 'bucket.\n"%s" is not valid.' % bucket_url) if (not BUCKET_NAME_RE.match(bucket_url.bucket_name) or TOO_LONG_DNS_NAME_COMP.search(bucket_url.bucket_name)): raise InvalidUrlError('Invalid bucket name in URL "%s"' % bucket_url.bucket_name) self.logger.info('Creating %s...', bucket_url) # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) try: self.gsutil_api.CreateBucket(bucket_url.bucket_name, project_id=self.project_id, metadata=bucket_metadata, provider=bucket_url.scheme) except AccessDeniedException as e: message = e.reason if 'key' in message: # This will print the error reason and append the following as a # suggested next step: # # To authorize, run: # gsutil kms authorize \ # -k <kms_key> \ # -p <project_id> message += ' To authorize, run:\n gsutil kms authorize' message += ' \\\n -k %s' % kms_key if (self.project_id): message += ' \\\n -p %s' % self.project_id raise CommandException(message) else: raise except BadRequestException as e: if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and bucket_url.scheme == 'gs'): bucket_name = bucket_url.bucket_name final_comp = bucket_name[bucket_name.rfind('.') + 1:] raise CommandException('\n'.join( textwrap.wrap( 'Buckets with "." in the name must be valid DNS names. The bucket' ' you are attempting to create (%s) is not a valid DNS name,' ' because the final component (%s) is not currently a valid part' ' of the top-level DNS tree.' % (bucket_name, final_comp)))) else: raise return 0	16.76 KB	2025-06-25 08:19
📄 mv.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like mv command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.commands.cp import CP_AND_MV_SHIM_FLAG_MAP from gslib.commands.cp import CP_SUB_ARGS from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil mv [-p] src_url dst_url gsutil mv [-p] src_url... dst_url gsutil mv [-p] -I dst_url """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``gsutil mv`` command allows you to move data between your local file system and the cloud, move data within the cloud, and move data between cloud storage providers. For example, to move all objects from a bucket to a local directory you could use: gsutil mv gs://my_bucket/* dir Similarly, to move all objects from a local directory to a bucket you could use: gsutil mv ./dir gs://my_bucket <B>RENAMING GROUPS OF OBJECTS</B> You can use the ``gsutil mv`` command to rename all objects with a given prefix to have a new prefix. For example, the following command renames all objects under gs://my_bucket/oldprefix to be under gs://my_bucket/newprefix, otherwise preserving the naming structure: gsutil mv gs://my_bucket/oldprefix gs://my_bucket/newprefix If you do a rename as specified above and you want to preserve ACLs, you should use the ``-p`` option (see OPTIONS). If you have a large number of files to move you might want to use the ``gsutil -m`` option, to perform a multi-threaded/multi-processing move: gsutil -m mv gs://my_bucket/oldprefix gs://my_bucket/newprefix <B>NON-ATOMIC OPERATION</B> Unlike the case with many file systems, the gsutil mv command does not perform a single atomic operation. Rather, it performs a copy from source to destination followed by removing the source for each object. A consequence of this is that, in addition to normal network and operation charges, if you move a Nearline Storage, Coldline Storage, or Archive Storage object, deletion and data retrieval charges apply. See the `documentation <https://cloud.google.com/storage/pricing>`_ for pricing details. <B>OPTIONS</B> All options that are available for the gsutil cp command are also available for the gsutil mv command (except for the -R flag, which is implied by the ``gsutil mv`` command). Please see the `Options section for cp <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for more information. """) MV_SHIM_FLAG_MAP = {*CP_AND_MV_SHIM_FLAG_MAP, '-r': None, '-R': None} class MvCommand(Command): """Implementation of gsutil mv command. Note that there is no atomic rename operation - this command is simply a shorthand for 'cp' followed by 'rm'. """ # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'mv', command_name_aliases=['move', 'ren', 'rename'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, # Flags for mv are passed through to cp. supported_sub_args=CP_SUB_ARGS, file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument() ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='mv', help_name_aliases=['move', 'rename'], help_type='command_help', help_one_line_summary='Move/rename objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'mv'], flag_map=MV_SHIM_FLAG_MAP, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the mv command.""" # Check each source arg up, refusing to delete a bucket src URL (force users # to explicitly do that as a separate operation). for arg_to_check in self.args[0:-1]: url = StorageUrlFromString(arg_to_check) if url.IsCloudUrl() and (url.IsBucket() or url.IsProvider()): raise CommandException('You cannot move a source bucket using the mv ' 'command. If you meant to move\nall objects in ' 'the bucket, you can use a command like:\n' '\tgsutil mv %s/ %s' % (arg_to_check, self.args[-1])) # Insert command-line opts in front of args so they'll be picked up by cp # and rm commands (e.g., for -p option). Use undocumented (internal # use-only) cp -M option, which causes each original object to be deleted # after successfully copying to its destination, and also causes naming # behavior consistent with Unix mv naming behavior (see comments in # ConstructDstUrl). unparsed_args = ['-M'] if self.recursion_requested: unparsed_args.append('-R') unparsed_args.extend(self.unparsed_args) self.command_runner.RunNamedCommand( 'cp', args=unparsed_args, headers=self.headers, debug=self.debug, trace_token=self.trace_token, user_project=self.user_project, parallel_operations=self.parallel_operations) return 0	6.42 KB	2025-06-25 08:19
📄 notification.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the notification command to gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import getopt import re import time import uuid from datetime import datetime from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PublishPermissionDeniedException from gslib.command import Command from gslib.command import NO_MAX from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.help_provider import CreateHelpText from gslib.project_id import PopulateProjectId from gslib.pubsub_api import PubsubApi from gslib.storage_url import StorageUrlFromString from gslib.third_party.pubsub_apitools.pubsub_v1_messages import Binding from gslib.utils import copy_helper from gslib.utils import shim_util from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap # Cloud Pub/Sub commands _LIST_SYNOPSIS = """ gsutil notification list gs://<bucket_name>... """ _DELETE_SYNOPSIS = """ gsutil notification delete (<notificationConfigName>\|gs://<bucket_name>)... """ _CREATE_SYNOPSIS = """ gsutil notification create -f (json\|none) [-p <prefix>] [-t <topic>] \\ [-m <key>:<value>]... [-e <eventType>]... gs://<bucket_name> """ # Object Change Notification commands _WATCHBUCKET_SYNOPSIS = """ gsutil notification watchbucket [-i <id>] [-t <token>] <app_url> gs://<bucket_name> """ _STOPCHANNEL_SYNOPSIS = """ gsutil notification stopchannel <channel_id> <resource_id> """ _SYNOPSIS = ( _CREATE_SYNOPSIS + _DELETE_SYNOPSIS.lstrip('\n') + _LIST_SYNOPSIS.lstrip('\n') + _WATCHBUCKET_SYNOPSIS + _STOPCHANNEL_SYNOPSIS.lstrip('\n') + '\n') # yapf: disable _LIST_DESCRIPTION = """ <B>LIST</B> The list sub-command provides a list of notification configs belonging to a given bucket. The listed name of each notification config can be used with the delete sub-command to delete that specific notification config. For listing Object Change Notifications instead of Cloud Pub/Sub notification subscription configs, add a -o flag. <B>LIST EXAMPLES</B> Fetch the list of notification configs for the bucket example-bucket: gsutil notification list gs://example-bucket The same as above, but for Object Change Notifications instead of Cloud Pub/Sub notification subscription configs: gsutil notification list -o gs://example-bucket Fetch the notification configs in all buckets matching a wildcard: gsutil notification list gs://example-* Fetch all of the notification configs for buckets in the default project: gsutil notification list gs://* """ _DELETE_DESCRIPTION = """ <B>DELETE</B> The delete sub-command deletes notification configs from a bucket. If a notification config name is passed as a parameter, that notification config alone is deleted. If a bucket name is passed, all notification configs associated with that bucket are deleted. Cloud Pub/Sub topics associated with this notification config are not deleted by this command. Those must be deleted separately, for example with the gcloud command `gcloud beta pubsub topics delete`. Object Change Notification subscriptions cannot be deleted with this command. For that, see the command `gsutil notification stopchannel`. <B>DELETE EXAMPLES</B> Delete a single notification config (with ID 3) in the bucket example-bucket: gsutil notification delete projects/_/buckets/example-bucket/notificationConfigs/3 Delete all notification configs in the bucket example-bucket: gsutil notification delete gs://example-bucket """ _CREATE_DESCRIPTION = """ <B>CREATE</B> The create sub-command creates a notification config on a bucket, establishing a flow of event notifications from Cloud Storage to a Cloud Pub/Sub topic. As part of creating this flow, the create command also verifies that the destination Cloud Pub/Sub topic exists, creating it if necessary, and verifies that the Cloud Storage bucket has permission to publish events to that topic, granting the permission if necessary. If a destination Cloud Pub/Sub topic is not specified with the -t flag, Cloud Storage chooses a topic name in the default project whose ID is the same as the bucket name. For example, if the default project ID specified is 'default-project' and the bucket being configured is gs://example-bucket, the create command uses the Cloud Pub/Sub topic "projects/default-project/topics/example-bucket". In order to enable notifications, your project's `Cloud Storage service agent <https://cloud.google.com/storage/docs/projects#service-accounts>`_ must have the IAM permission "pubsub.topics.publish". This command checks to see if the destination Cloud Pub/Sub topic grants the service agent this permission. If not, the create command attempts to grant it. A bucket can have up to 100 total notification configurations and up to 10 notification configurations set to trigger for a specific event. <B>CREATE EXAMPLES</B> Begin sending notifications of all changes to the bucket example-bucket to the Cloud Pub/Sub topic projects/default-project/topics/example-bucket: gsutil notification create -f json gs://example-bucket The same as above, but specifies the destination topic ID 'files-to-process' in the default project: gsutil notification create -f json \\ -t files-to-process gs://example-bucket The same as above, but specifies a Cloud Pub/Sub topic belonging to the specific cloud project 'example-project': gsutil notification create -f json \\ -t projects/example-project/topics/files-to-process gs://example-bucket Create a notification config that only sends an event when a new object has been created: gsutil notification create -f json -e OBJECT_FINALIZE gs://example-bucket Create a topic and notification config that only sends an event when an object beginning with "photos/" is affected: gsutil notification create -p photos/ gs://example-bucket List all of the notificationConfigs in bucket example-bucket: gsutil notification list gs://example-bucket Delete all notitificationConfigs for bucket example-bucket: gsutil notification delete gs://example-bucket Delete one specific notificationConfig for bucket example-bucket: gsutil notification delete \\ projects/_/buckets/example-bucket/notificationConfigs/1 <B>OPTIONS</B> The create sub-command has the following options -e Specify an event type filter for this notification config. Cloud Storage only sends notifications of this type. You may specify this parameter multiple times to allow multiple event types. If not specified, Cloud Storage sends notifications for all event types. The valid types are: OBJECT_FINALIZE - An object has been created. OBJECT_METADATA_UPDATE - The metadata of an object has changed. OBJECT_DELETE - An object has been permanently deleted. OBJECT_ARCHIVE - A live version of an object has become a noncurrent version. -f Specifies the payload format of notification messages. Must be either "json" for a payload matches the object metadata for the JSON API, or "none" to specify no payload at all. In either case, notification details are available in the message attributes. -m Specifies a key:value attribute that is appended to the set of attributes sent to Cloud Pub/Sub for all events associated with this notification config. You may specify this parameter multiple times to set multiple attributes. -p Specifies a prefix path filter for this notification config. Cloud Storage only sends notifications for objects in this bucket whose names begin with the specified prefix. -s Skips creation and permission assignment of the Cloud Pub/Sub topic. This is useful if the caller does not have permission to access the topic in question, or if the topic already exists and has the appropriate publish permission assigned. -t The Cloud Pub/Sub topic to which notifications should be sent. If not specified, this command chooses a topic whose project is your default project and whose ID is the same as the Cloud Storage bucket name. <B>NEXT STEPS</B> Once the create command has succeeded, Cloud Storage publishes a message to the specified Cloud Pub/Sub topic when eligible changes occur. In order to receive these messages, you must create a Pub/Sub subscription for your Pub/Sub topic. To learn more about creating Pub/Sub subscriptions, see `the Pub/Sub Subscriber Overview <https://cloud.google.com/pubsub/docs/subscriber>`_. You can create a simple Pub/Sub subscription using the ``gcloud`` command-line tool. For example, to create a new subscription on the topic "myNewTopic" and attempt to pull messages from it, you could run: gcloud beta pubsub subscriptions create --topic myNewTopic testSubscription gcloud beta pubsub subscriptions pull --auto-ack testSubscription """ _WATCHBUCKET_DESCRIPTION = """ <B>WATCHBUCKET</B> The watchbucket sub-command can be used to watch a bucket for object changes. A service account must be used when running this command. The app_url parameter must be an HTTPS URL to an application that will be notified of changes to any object in the bucket. The optional id parameter can be used to assign a unique identifier to the created notification channel. If not provided, a random UUID string is generated. The optional token parameter can be used to validate notifications events. To do this, set this custom token and store it to later verify that notification events contain the client token you expect. <B>WATCHBUCKET EXAMPLES</B> Watch the bucket example-bucket for changes and send notifications to an application server running at example.com: gsutil notification watchbucket https://example.com/notify \\ gs://example-bucket Assign identifier my-channel-id to the created notification channel: gsutil notification watchbucket -i my-channel-id \\ https://example.com/notify gs://example-bucket Set a custom client token that is included with each notification event: gsutil notification watchbucket -t my-client-token \\ https://example.com/notify gs://example-bucket """ _STOPCHANNEL_DESCRIPTION = """ <B>STOPCHANNEL</B> The stopchannel sub-command can be used to stop sending change events to a notification channel. The channel_id and resource_id parameters should match the values from the response of a bucket watch request. <B>STOPCHANNEL EXAMPLES</B> Stop the notification event channel with channel identifier channel1 and resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8: gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8 """ _DESCRIPTION = """ You can use the ``notification`` command to configure `Pub/Sub notifications for Cloud Storage <https://cloud.google.com/storage/docs/pubsub-notifications>`_ and `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_ channels. <B>CLOUD PUB/SUB</B> The "create", "list", and "delete" sub-commands deal with configuring Cloud Storage integration with Google Cloud Pub/Sub. """ + _CREATE_DESCRIPTION + _LIST_DESCRIPTION + _DELETE_DESCRIPTION + """ <B>OBJECT CHANGE NOTIFICATIONS</B> Object change notification is a separate, older feature within Cloud Storage for generating notifications. This feature sends HTTPS messages to a client application that you've set up separately. This feature is generally not recommended, because Pub/Sub notifications are cheaper, easier to use, and more flexible. For more information, see `Object change notification <https://cloud.google.com/storage/docs/object-change-notification>`_. The "watchbucket" and "stopchannel" sub-commands enable and disable Object change notifications. """ + _WATCHBUCKET_DESCRIPTION + _STOPCHANNEL_DESCRIPTION + """ <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B> gsutil supports `parallel composite uploads <https://cloud.google.com/storage/docs/uploads-downloads#parallel-composite-uploads>`_. If enabled, an upload can result in multiple temporary component objects being uploaded before the actual intended object is created. Any subscriber to notifications for this bucket then sees a notification for each of these components being created and deleted. If this is a concern for you, note that parallel composite uploads can be disabled by setting "parallel_composite_upload_threshold = 0" in your .boto config file. Alternately, your subscriber code can filter out gsutil's parallel composite uploads by ignoring any notification about objects whose names contain (but do not start with) the following string: "{composite_namespace}". """.format(composite_namespace=copy_helper.PARALLEL_UPLOAD_TEMP_NAMESPACE) NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE = """ Watch bucket attempt failed: {watch_error} You attempted to watch a bucket with an application URL of: {watch_url} which is not authorized for your project. Please ensure that you are using Service Account authentication and that the Service Account's project is authorized for the application URL. Notification endpoint URLs must also be whitelisted in your Cloud Console project. To do that, the domain must also be verified using Google Webmaster Tools. For instructions, please see `Notification Authorization <https://cloud.google.com/storage/docs/object-change-notification#_Authorization>`_. """ _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) # yapf: disable _create_help_text = ( CreateHelpText(_CREATE_SYNOPSIS, _CREATE_DESCRIPTION)) _list_help_text = ( CreateHelpText(_LIST_SYNOPSIS, _LIST_DESCRIPTION)) _delete_help_text = ( CreateHelpText(_DELETE_SYNOPSIS, _DELETE_DESCRIPTION)) _watchbucket_help_text = ( CreateHelpText(_WATCHBUCKET_SYNOPSIS, _WATCHBUCKET_DESCRIPTION)) _stopchannel_help_text = ( CreateHelpText(_STOPCHANNEL_SYNOPSIS, _STOPCHANNEL_DESCRIPTION)) # yapf: enable PAYLOAD_FORMAT_MAP = { 'none': 'NONE', 'json': 'JSON_API_V1', } class NotificationCommand(Command): """Implementation of gsutil notification command.""" # Notification names might look like one of these: # canonical form: projects/_/buckets/bucket/notificationConfigs/3 # JSON API form: b/bucket/notificationConfigs/5 # Either of the above might start with a / if a user is copying & pasting. def _GetNotificationPathRegex(self): if not NotificationCommand._notification_path_regex: NotificationCommand._notification_path_regex = re.compile( ('/?(projects/[^/]+/)?b(uckets)?/(?P<bucket>[^/]+)/' 'notificationConfigs/(?P<notification>[0-9]+)')) return NotificationCommand._notification_path_regex _notification_path_regex = None # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'notification', command_name_aliases=[ 'notify', 'notifyconfig', 'notifications', 'notif', ], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='i:t:m:t:of:e:p:s', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'watchbucket': [ CommandArgument.MakeFreeTextArgument(), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'stopchannel': [], 'list': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], 'delete': [ # Takes a list of one of the following: # notification: projects/_/buckets/bla/notificationConfigs/5, # bucket: gs://foobar CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], 'create': [ CommandArgument.MakeFreeTextArgument(), # Cloud Pub/Sub topic CommandArgument.MakeNCloudBucketURLsArgument(1), ] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='notification', help_name_aliases=[ 'watchbucket', 'stopchannel', 'notifyconfig', ], help_type='command_help', help_one_line_summary='Configure object change notification', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'create': _create_help_text, 'list': _list_help_text, 'delete': _delete_help_text, 'watchbucket': _watchbucket_help_text, 'stopchannel': _stopchannel_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'create': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'create' ], flag_map={ '-m': GcloudStorageFlag( '--custom-attributes', repeat_type=shim_util.RepeatFlagType.DICT), '-e': GcloudStorageFlag( '--event-types', repeat_type=shim_util.RepeatFlagType.LIST), '-p': GcloudStorageFlag('--object-prefix'), '-f': GcloudStorageFlag('--payload-format'), '-s': GcloudStorageFlag('--skip-topic-setup'), '-t': GcloudStorageFlag('--topic'), }, ), 'delete': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'delete' ], flag_map={}, ), 'list': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'notifications', 'list', '--human-readable', ], flag_map={}, supports_output_translation=True, ), }, flag_map={}, ) def _WatchBucket(self): """Creates a watch on a bucket given in self.args.""" self.CheckArguments() identifier = None client_token = None if self.sub_opts: for o, a in self.sub_opts: if o == '-i': identifier = a if o == '-t': client_token = a identifier = identifier or str(uuid.uuid4()) watch_url = self.args[0] bucket_arg = self.args[-1] if not watch_url.lower().startswith('https://'): raise CommandException('The application URL must be an https:// URL.') bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) self.logger.info('Watching bucket %s with application URL %s ...', bucket_url, watch_url) try: channel = self.gsutil_api.WatchBucket(bucket_url.bucket_name, watch_url, identifier, token=client_token, provider=bucket_url.scheme) except AccessDeniedException as e: self.logger.warn( NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(watch_error=str(e), watch_url=watch_url)) raise channel_id = channel.id resource_id = channel.resourceId client_token = channel.token self.logger.info('Successfully created watch notification channel.') self.logger.info('Watch channel identifier: %s', channel_id) self.logger.info('Canonicalized resource identifier: %s', resource_id) self.logger.info('Client state token: %s', client_token) return 0 def _StopChannel(self): channel_id = self.args[0] resource_id = self.args[1] self.logger.info('Removing channel %s with resource identifier %s ...', channel_id, resource_id) self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs') self.logger.info('Succesfully removed channel.') return 0 def _ListChannels(self, bucket_arg): """Lists active channel watches on a bucket given in self.args.""" bucket_url = StorageUrlFromString(bucket_arg) if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'): raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) if not bucket_url.IsBucket(): raise CommandException('URL must name a bucket for the %s command.' % self.command_name) channels = self.gsutil_api.ListChannels(bucket_url.bucket_name, provider='gs').items self.logger.info( 'Bucket %s has the following active Object Change Notifications:', bucket_url.bucket_name) for idx, channel in enumerate(channels): self.logger.info('\tNotification channel %d:', idx + 1) self.logger.info('\t\tChannel identifier: %s', channel.channel_id) self.logger.info('\t\tResource identifier: %s', channel.resource_id) self.logger.info('\t\tApplication URL: %s', channel.push_url) self.logger.info('\t\tCreated by: %s', channel.subscriber_email) self.logger.info( '\t\tCreation time: %s', str(datetime.fromtimestamp(channel.creation_time_ms / 1000))) return 0 def _Create(self): self.CheckArguments() # User-specified options pubsub_topic = None payload_format = None custom_attributes = {} event_types = [] object_name_prefix = None should_setup_topic = True if self.sub_opts: for o, a in self.sub_opts: if o == '-e': event_types.append(a) elif o == '-f': payload_format = a elif o == '-m': if ':' not in a: raise CommandException( 'Custom attributes specified with -m should be of the form ' 'key:value') key, value = a.split(':', 1) custom_attributes[key] = value elif o == '-p': object_name_prefix = a elif o == '-s': should_setup_topic = False elif o == '-t': pubsub_topic = a if payload_format not in PAYLOAD_FORMAT_MAP: raise CommandException( "Must provide a payload format with -f of either 'json' or 'none'") payload_format = PAYLOAD_FORMAT_MAP[payload_format] bucket_arg = self.args[-1] bucket_url = StorageUrlFromString(bucket_arg) if not bucket_url.IsCloudUrl() or not bucket_url.IsBucket(): raise CommandException( "%s %s requires a GCS bucket name, but got '%s'" % (self.command_name, self.subcommand_name, bucket_arg)) if bucket_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) bucket_name = bucket_url.bucket_name self.logger.debug('Creating notification for bucket %s', bucket_url) # Find the project this bucket belongs to bucket_metadata = self.gsutil_api.GetBucket(bucket_name, fields=['projectNumber'], provider=bucket_url.scheme) bucket_project_number = bucket_metadata.projectNumber # If not specified, choose a sensible default for the Cloud Pub/Sub topic # name. if not pubsub_topic: pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), bucket_name) if not pubsub_topic.startswith('projects/'): # If a user picks a topic ID (mytopic) but doesn't pass the whole name ( # projects/my-project/topics/mytopic ), pick a default project. pubsub_topic = 'projects/%s/topics/%s' % (PopulateProjectId(None), pubsub_topic) self.logger.debug('Using Cloud Pub/Sub topic %s', pubsub_topic) just_modified_topic_permissions = False if should_setup_topic: # Ask GCS for the email address that represents GCS's permission to # publish to a Cloud Pub/Sub topic from this project. service_account = self.gsutil_api.GetProjectServiceAccount( bucket_project_number, provider=bucket_url.scheme).email_address self.logger.debug('Service account for project %d: %s', bucket_project_number, service_account) just_modified_topic_permissions = self._CreateTopic( pubsub_topic, service_account) for attempt_number in range(0, 2): try: create_response = self.gsutil_api.CreateNotificationConfig( bucket_name, pubsub_topic=pubsub_topic, payload_format=payload_format, custom_attributes=custom_attributes, event_types=event_types if event_types else None, object_name_prefix=object_name_prefix, provider=bucket_url.scheme) break except PublishPermissionDeniedException: if attempt_number == 0 and just_modified_topic_permissions: # If we have just set the IAM policy, it may take up to 10 seconds to # take effect. self.logger.info( 'Retrying create notification in 10 seconds ' '(new permissions may take up to 10 seconds to take effect.)') time.sleep(10) else: raise notification_name = 'projects/_/buckets/%s/notificationConfigs/%s' % ( bucket_name, create_response.id) self.logger.info('Created notification config %s', notification_name) return 0 def _CreateTopic(self, pubsub_topic, service_account): """Assures that a topic exists, creating it if necessary. Also adds GCS as a publisher on that bucket, if necessary. Args: pubsub_topic: name of the Cloud Pub/Sub topic to use/create. service_account: the GCS service account that needs publish permission. Returns: true if we modified IAM permissions, otherwise false. """ pubsub_api = PubsubApi(logger=self.logger) # Verify that the Pub/Sub topic exists. If it does not, create it. try: pubsub_api.GetTopic(topic_name=pubsub_topic) self.logger.debug('Topic %s already exists', pubsub_topic) except NotFoundException: self.logger.debug('Creating topic %s', pubsub_topic) pubsub_api.CreateTopic(topic_name=pubsub_topic) self.logger.info('Created Cloud Pub/Sub topic %s', pubsub_topic) # Verify that the service account is in the IAM policy. policy = pubsub_api.GetTopicIamPolicy(topic_name=pubsub_topic) binding = Binding(role='roles/pubsub.publisher', members=['serviceAccount:%s' % service_account]) # This could be more extensive. We could, for instance, check for roles # that are stronger that pubsub.publisher, like owner. We could also # recurse up the hierarchy looking to see if there are project-level # permissions. This can get very complex very quickly, as the caller # may not necessarily have access to the project-level IAM policy. # There's no danger in double-granting permission just to make sure it's # there, though. if binding not in policy.bindings: policy.bindings.append(binding) # transactional safety via etag field. pubsub_api.SetTopicIamPolicy(topic_name=pubsub_topic, policy=policy) return True else: self.logger.debug('GCS already has publish permission to topic %s.', pubsub_topic) return False def _EnumerateNotificationsFromArgs(self, accept_notification_configs=True): """Yields bucket/notification tuples from command-line args. Given a list of strings that are bucket names (gs://foo) or notification config IDs, yield tuples of bucket names and their associated notifications. Args: accept_notification_configs: whether notification configs are valid args. Yields: Tuples of the form (bucket_name, Notification) """ path_regex = self._GetNotificationPathRegex() for list_entry in self.args: match = path_regex.match(list_entry) if match: if not accept_notification_configs: raise CommandException( '%s %s accepts only bucket names, but you provided %s' % (self.command_name, self.subcommand_name, list_entry)) bucket_name = match.group('bucket') notification_id = match.group('notification') found = False for notification in self.gsutil_api.ListNotificationConfigs( bucket_name, provider='gs'): if notification.id == notification_id: yield (bucket_name, notification) found = True break if not found: raise NotFoundException('Could not find notification %s' % list_entry) else: storage_url = StorageUrlFromString(list_entry) if not storage_url.IsCloudUrl(): raise CommandException( 'The %s command must be used on cloud buckets or notification ' 'config names.' % self.command_name) if storage_url.scheme != 'gs': raise CommandException('The %s command only works on gs:// buckets.') path = None if storage_url.IsProvider(): path = 'gs://*' elif storage_url.IsBucket(): path = list_entry if not path: raise CommandException( 'The %s command cannot be used on cloud objects, only buckets' % self.command_name) for blr in self.WildcardIterator(path).IterBuckets( bucket_fields=['id']): for notification in self.gsutil_api.ListNotificationConfigs( blr.storage_url.bucket_name, provider='gs'): yield (blr.storage_url.bucket_name, notification) def _List(self): self.CheckArguments() if self.sub_opts: if '-o' in dict(self.sub_opts): for bucket_name in self.args: self._ListChannels(bucket_name) else: for bucket_name, notification in self._EnumerateNotificationsFromArgs( accept_notification_configs=False): self._PrintNotificationDetails(bucket_name, notification) return 0 def _PrintNotificationDetails(self, bucket, notification): print('projects/_/buckets/{bucket}/notificationConfigs/{notification}\n' '\tCloud Pub/Sub topic: {topic}'.format( bucket=bucket, notification=notification.id, topic=notification.topic[len('//pubsub.googleapis.com/'):])) if notification.custom_attributes: print('\tCustom attributes:') for attr in notification.custom_attributes.additionalProperties: print('\t\t%s: %s' % (attr.key, attr.value)) filters = [] if notification.event_types: filters.append('\t\tEvent Types: %s' % ', '.join(notification.event_types)) if notification.object_name_prefix: filters.append("\t\tObject name prefix: '%s'" % notification.object_name_prefix) if filters: print('\tFilters:') for line in filters: print(line) self.logger.info('') def _Delete(self): for bucket_name, notification in self._EnumerateNotificationsFromArgs(): self._DeleteNotification(bucket_name, notification.id) return 0 def _DeleteNotification(self, bucket_name, notification_id): self.gsutil_api.DeleteNotificationConfig(bucket_name, notification=notification_id, provider='gs') return 0 def _RunSubCommand(self, func): try: (self.sub_opts, self.args) = getopt.getopt(self.args, self.command_spec.supported_sub_args) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(sub_opts=self.sub_opts) return func(self) except getopt.GetoptError: self.RaiseInvalidArgumentException() SUBCOMMANDS = { 'create': _Create, 'list': _List, 'delete': _Delete, 'watchbucket': _WatchBucket, 'stopchannel': _StopChannel } def RunCommand(self): """Command entry point for the notification command.""" self.subcommand_name = self.args.pop(0) if self.subcommand_name in NotificationCommand.SUBCOMMANDS: metrics.LogCommandParams(subcommands=[self.subcommand_name]) return self._RunSubCommand( NotificationCommand.SUBCOMMANDS[self.subcommand_name]) else: raise CommandException('Invalid subcommand "%s" for the %s command.' % (self.subcommand_name, self.command_name))	33.94 KB	2025-06-25 08:19
📄 pap.py # -- coding: utf-8 -- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the pap command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil pap set (enforced\|inherited) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil pap get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``pap set`` command configures public access prevention for Cloud Storage buckets. If you set a bucket to be ``inherited``, it uses public access prevention only if the bucket is subject to the `public access prevention <https://cloud.google.com/storage/docs/org-policy-constraints#public-access-prevention>`_ organization policy constraint. <B>SET EXAMPLES</B> Configure ``redbucket`` and ``bluebucket`` to use public access prevention: gsutil pap set enforced gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``pap get`` command returns public access prevention values for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if ``redbucket`` and ``bluebucket`` are using public access prevention: gsutil pap get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``pap`` command is used to retrieve or configure the `public access prevention <https://cloud.google.com/storage/docs/public-access-prevention>`_ setting of Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit enforced one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue _GCLOUD_LIST_FORMAT = ('--format=value[separator=": "]' + '(name.sub("' + shim_util.get_format_flag_caret() + '","gs://"),' + 'iamConfiguration.publicAccessPrevention.yesno(' + 'no="inherited"))') class PapCommand(Command): """Implements the gsutil pap command.""" command_spec = Command.CreateCommandSpec( 'pap', command_name_aliases=['publicaccessprevention'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['enforced', 'inherited']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='pap', help_name_aliases=['publicaccessprevention'], help_type='command_help', help_one_line_summary='Configure public access prevention', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_LIST_FORMAT, '--raw' ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'enforced': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--public-access-prevention', ], flag_map={}, ), 'inherited': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-public-access-prevention', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetPublicAccessPrevention(self, blr): """Gets the public access prevention setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration public_access_prevention = iam_config.publicAccessPrevention or 'inherited' bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, public_access_prevention)) def _SetPublicAccessPrevention(self, blr, setting_arg): """Sets the Public Access Prevention setting for a bucket enforced or inherited.""" bucket_url = blr.storage_url iam_config = IamConfigurationValue() iam_config.publicAccessPrevention = setting_arg bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) print('Setting Public Access Prevention %s for %s' % (setting_arg, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Pap(self): """Handles pap command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('pap only supports get\|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetPublicAccessPrevention elif subcommand == 'set': subcommand_func = self._SetPublicAccessPrevention setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'Buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the pap command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Pap() else: raise CommandException('Invalid subcommand "%s", use get\|set instead.' % action_subcommand)	9.25 KB	2025-06-25 08:19
📄 perfdiag.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains the perfdiag gsutil command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar from collections import defaultdict from collections import namedtuple import contextlib import datetime import json import logging import math import multiprocessing import os import random import re import socket import string import subprocess import tempfile import time import boto import boto.gs.connection import six from six.moves import cStringIO from six.moves import http_client from six.moves import range from six.moves import range import gslib from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.command_argument import CommandArgument from gslib.commands import config from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.file_part import FilePart from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import text_util from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import ResumableThreshold from gslib.utils.constants import UTF8 from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.cloud_api_helper import GetDownloadSerializationData from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.system_util import CheckFreeSpace from gslib.utils.system_util import GetDiskCounters from gslib.utils.system_util import GetFileSize from gslib.utils.system_util import IS_LINUX from gslib.utils.system_util import IsRunningInCiEnvironment from gslib.utils.unit_util import DivideAndCeil from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeBitsHumanReadable from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import Percentile _SYNOPSIS = """ gsutil perfdiag [-i in.json] gsutil perfdiag [-o out.json] [-n objects] [-c processes] [-k threads] [-p parallelism type] [-y slices] [-s size] [-d directory] [-t tests] [-j ratio] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The ``perfdiag`` command runs a suite of diagnostic tests for a given Cloud Storage bucket. The ``bucket_name`` parameter must name an existing bucket to which the user has write permission. Several test files will be uploaded to and downloaded from this bucket. All test files will be deleted at the completion of the diagnostic if it finishes successfully. For a list of relevant permissions, see `Cloud IAM permissions for gsutil commands <https://cloud.google.com/storage/docs/access-control/iam-gsutil>`_. gsutil performance can be influenced by a number of factors originating at the client, server, or network level. Some examples include the following: + CPU speed + Available memory + The access path to the local disk + Network bandwidth + Contention and error rates along the path between gsutil and Google servers + Operating system buffering configuration + Firewalls and other network elements The `perfdiag` command is provided so that customers can run a known measurement suite when troubleshooting performance problems. <B>PROVIDING DIAGNOSTIC OUTPUT TO THE CLOUD STORAGE TEAM</B> If the Cloud Storage team asks you to run a performance diagnostic please use the following command, and email the output file (output.json) to the @google.com address provided by the Cloud Storage team: gsutil perfdiag -o output.json gs://your-bucket Additional resources for discussing ``perfdiag`` results include the `Stack Overflow tag for Cloud Storage <https://stackoverflow.com/questions/tagged/google-cloud-storage>`_ and the `gsutil GitHub repository <https://github.com/GoogleCloudPlatform/gsutil/issues>`_. <B>OPTIONS</B> -n Sets the number of objects to use when downloading and uploading files during tests. Defaults to 5. -c Sets the number of `processes <https://en.wikipedia.org/wiki/Process_(computing)>`_ to use while running throughput experiments. The default value is 1. -k Sets the number of `threads <https://en.wikipedia.org/wiki/Thread_(computing)>`_ per process to use while running throughput experiments. Each process will receive an equal number of threads. The default value is 1. NOTE: All specified threads and processes will be created, but may not by saturated with work if too few objects (specified with ``-n``) and too few components (specified with ``-y``) are specified. -p Sets the type of parallelism to be used (only applicable when threads or processes are specified and threads * processes > 1). The default is to use ``fan``. Must be one of the following: fan Use one thread per object. This is akin to using gsutil ``-m cp``, with sliced object download / parallel composite upload disabled. slice Use Y (specified with ``-y``) threads for each object, transferring one object at a time. This is akin to using parallel object download / parallel composite upload, without ``-m``. Sliced uploads not supported for s3. both Use Y (specified with ``-y``) threads for each object, transferring multiple objects at a time. This is akin to simultaneously using sliced object download / parallel composite upload and ``gsutil -m cp``. Parallel composite uploads not supported for s3. -y Sets the number of slices to divide each file/object into while transferring data. Only applicable with the slice (or both) parallelism type. The default is 4 slices. -s Sets the size (in bytes) for each of the N (set with ``-n``) objects used in the read and write throughput tests. The default is 1 MiB. This can also be specified using byte suffixes such as 500K or 1M. NOTE: these values are interpreted as multiples of 1024 (K=1024, M=10241024, etc.) NOTE: If ``rthru_file`` or ``wthru_file`` are performed, N (set with ``-n``) times as much disk space as specified will be required for the operation. -d Sets the directory to store temporary local files in. If not specified, a default temporary directory will be used. -t Sets the list of diagnostic tests to perform. The default is to run the ``lat``, ``rthru``, and ``wthru`` diagnostic tests. Must be a comma-separated list containing one or more of the following: lat For N (set with ``-n``) objects, write the object, retrieve its metadata, read the object, and finally delete the object. Record the latency of each operation. list Write N (set with ``-n``) objects to the bucket, record how long it takes for the eventually consistent listing call to return the N objects in its result, delete the N objects, then record how long it takes listing to stop returning the N objects. rthru Runs N (set with ``-n``) read operations, with at most C (set with -c) reads outstanding at any given time. rthru_file The same as ``rthru``, but simultaneously writes data to the disk, to gauge the performance impact of the local disk on downloads. wthru Runs N (set with ``-n``) write operations, with at most C (set with ``-c``) writes outstanding at any given time. wthru_file The same as wthru, but simultaneously reads data from the disk, to gauge the performance impact of the local disk on uploads. -m Adds metadata to the result JSON file. Multiple ``-m`` values can be specified. Example: gsutil perfdiag -m "key1:val1" -m "key2:val2" gs://bucketname Each metadata key will be added to the top-level "metadata" dictionary in the output JSON file. -o Writes the results of the diagnostic to an output file. The output is a JSON file containing system information and performance diagnostic results. The file can be read and reported later using the ``-i`` option. -i Reads the JSON output file created using the ``-o`` command and prints a formatted description of the results. -j Applies gzip transport encoding and sets the target compression ratio for the generated test files. This ratio can be an integer between 0 and 100 (inclusive), with 0 generating a file with uniform data, and 100 generating random data. When you specify the ``-j`` option, files being uploaded are compressed in-memory and on-the-wire only. See `cp -j <https://cloud.google.com/storage/docs/gsutil/commands/cp#options>`_ for specific semantics. <B>MEASURING AVAILABILITY</B> The ``perfdiag`` command ignores the boto num_retries configuration parameter. Instead, it always retries on HTTP errors in the 500 range and keeps track of how many 500 errors were encountered during the test. The availability measurement is reported at the end of the test. Note that HTTP responses are only recorded when the request was made in a single process. When using multiple processes or threads, read and write throughput measurements are performed in an external process, so the availability numbers reported won't include the throughput measurements. <B>NOTE</B> The ``perfdiag`` command runs a series of tests that collects system information, such as the following: + Retrieves requester's IP address. + Executes DNS queries to Google servers and collects the results. + Collects network statistics information from the output of ``netstat -s`` and evaluates the BIOS product name string. + If a proxy server is configured, attempts to connect to it to retrieve the location and storage class of the bucket being used for performance testing. None of this information will be sent to Google unless you proactively choose to send it. """) FileDataTuple = namedtuple('FileDataTuple', 'size md5 data') # Describes one object in a fanned download. If need_to_slice is specified as # True, the object should be downloaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Download. FanDownloadTuple = namedtuple( 'FanDownloadTuple', 'need_to_slice object_name file_name serialization_data') # Describes one slice in a sliced download. # Field names are the same as documented in PerfDiagCommand.Download. SliceDownloadTuple = namedtuple( 'SliceDownloadTuple', 'object_name file_name serialization_data start_byte end_byte') # Describes one file in a fanned upload. If need_to_slice is specified as # True, the file should be uploaded with the slice strategy. Other field # names are the same as documented in PerfDiagCommand.Upload. FanUploadTuple = namedtuple( 'FanUploadTuple', 'need_to_slice file_name object_name use_file gzip_encoded') # Describes one slice in a sliced upload. # Field names are the same as documented in PerfDiagCommand.Upload. SliceUploadTuple = namedtuple( 'SliceUploadTuple', 'file_name object_name use_file file_start file_size gzip_encoded') # Dict storing file_path:FileDataTuple for each temporary file used by # perfdiag. This data should be kept outside of the PerfDiagCommand class # since calls to Apply will make copies of all member data. temp_file_dict = {} class Error(Exception): """Base exception class for this module.""" pass class InvalidArgument(Error): """Raised on invalid arguments to functions.""" pass def _DownloadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel downloads. Args: cls: The calling PerfDiagCommand class instance. args: A FanDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedDownload(args.object_name, args.file_name, args.serialization_data) else: cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data) def _DownloadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced downloads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceDownloadTuple object describing this download. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Download(args.object_name, gsutil_api, args.file_name, args.serialization_data, args.start_byte, args.end_byte) def _UploadObject(cls, args, thread_state=None): """Function argument to apply for performing fanned parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A FanUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) if args.need_to_slice: cls.PerformSlicedUpload(args.file_name, args.object_name, args.use_file, gsutil_api, gzip_encoded=args.gzip_encoded) else: cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, gzip_encoded=args.gzip_encoded) def _UploadSlice(cls, args, thread_state=None): """Function argument to apply for performing sliced parallel uploads. Args: cls: The calling PerfDiagCommand class instance. args: A SliceUploadTuple object describing this upload. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Upload(args.file_name, args.object_name, gsutil_api, args.use_file, args.file_start, args.file_size, gzip_encoded=args.gzip_encoded) def _DeleteWrapper(cls, object_name, thread_state=None): """Function argument to apply for performing parallel object deletions. Args: cls: The calling PerfDiagCommand class instance. object_name: The object name to delete from the test bucket. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(cls, thread_state) cls.Delete(object_name, gsutil_api) def _PerfdiagExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) def _DummyTrackerCallback(_): pass class DummyFile(object): """A dummy, file-like object that throws away everything written to it.""" # Because Python2 is so loose about bytes and text, Python3 code really # works best with a hint about a file object's mode. mode = 'bw' def write(self, args, *kwargs): # pylint: disable=invalid-name pass def close(self): # pylint: disable=invalid-name pass def _GenerateFileData(fp, file_size=0, random_ratio=100, max_unique_random_bytes=5242883): """Writes data into a file like object. Args: fp: A file like object to write the data to. file_size: The amount of data to write to the file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. max_unique_random_bytes: The maximum number of bytes to generate pseudo-randomly before beginning to repeat bytes. The default was chosen as the next prime larger than 5 MiB. """ # Normalize the ratio. random_ratio /= 100.0 random_bytes = os.urandom(min(file_size, max_unique_random_bytes)) total_bytes_written = 0 while total_bytes_written < file_size: num_bytes = min(max_unique_random_bytes, file_size - total_bytes_written) # Calculate the amount of random and sequential data to write and # resolve rounding errors by adjusting the random bytes to write. num_bytes_seq = int(num_bytes (1 - random_ratio)) num_bytes_random = num_bytes - num_bytes_seq fp.write(random_bytes[:num_bytes_random]) fp.write(b'x' * num_bytes_seq) total_bytes_written += num_bytes # Many functions in perfdiag re-define a temporary function based on a # variable from a loop, resulting in a false positive from the linter. # pylint: disable=cell-var-from-loop class PerfDiagCommand(Command): """Implementation of gsutil perfdiag command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'perfdiag', command_name_aliases=['diag', 'diagnostic', 'perf', 'performance'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='n:c:k:p:y:s:d:t:m:i:o:j:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudBucketURLsArgument(1)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='perfdiag', help_name_aliases=[], help_type='command_help', help_one_line_summary='Run performance diagnostic', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) # Byte sizes to use for latency testing files. # TODO: Consider letting the user specify these sizes with a configuration # parameter. test_lat_file_sizes = ( 0, # 0 bytes 1024, # 1 KiB 102400, # 100 KiB 1048576, # 1 MiB ) # Test names. RTHRU = 'rthru' RTHRU_FILE = 'rthru_file' WTHRU = 'wthru' WTHRU_FILE = 'wthru_file' LAT = 'lat' LIST = 'list' # Parallelism strategies. FAN = 'fan' SLICE = 'slice' BOTH = 'both' # List of all diagnostic tests. ALL_DIAG_TESTS = (RTHRU, RTHRU_FILE, WTHRU, WTHRU_FILE, LAT, LIST) # List of diagnostic tests to run by default. DEFAULT_DIAG_TESTS = (RTHRU, WTHRU, LAT) # List of parallelism strategies. PARALLEL_STRATEGIES = (FAN, SLICE, BOTH) # Google Cloud Storage XML API endpoint host. XML_API_HOST = boto.config.get('Credentials', 'gs_host', boto.gs.connection.GSConnection.DefaultHost) # Google Cloud Storage XML API endpoint port. XML_API_PORT = boto.config.getint('Credentials', 'gs_port', 80) # Maximum number of times to retry requests on 5xx errors. MAX_SERVER_ERROR_RETRIES = 5 # Maximum number of times to retry requests on more serious errors like # the socket breaking. MAX_TOTAL_RETRIES = 10 # The default buffer size in boto's Key object is set to 8 KiB. This becomes a # bottleneck at high throughput rates, so we increase it. KEY_BUFFER_SIZE = 16384 # Maximum amount of time, in seconds, we will wait for object listings to # reflect what we expect in the listing tests. MAX_LISTING_WAIT_TIME = 60.0 def _Exec(self, cmd, raise_on_error=True, return_output=False, mute_stderr=False): """Executes a command in a subprocess. Args: cmd: List containing the command to execute. raise_on_error: Whether or not to raise an exception when a process exits with a non-zero return code. return_output: If set to True, the return value of the function is the stdout of the process. mute_stderr: If set to True, the stderr of the process is not printed to the console. Returns: The return code of the process or the stdout if return_output is set. Raises: Exception: If raise_on_error is set to True and any process exits with a non-zero return code. """ self.logger.debug('Running command: %s', cmd) stderr = subprocess.PIPE if mute_stderr else None p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr) (stdoutdata, _) = p.communicate() if six.PY3: if isinstance(stdoutdata, bytes): stdoutdata = stdoutdata.decode(UTF8) if raise_on_error and p.returncode: raise CommandException("Received non-zero return code (%d) from " "subprocess '%s'." % (p.returncode, ' '.join(cmd))) return stdoutdata if return_output else p.returncode def _WarnIfLargeData(self): """Outputs a warning message if a large amount of data is being used.""" if self.num_objects * self.thru_filesize > HumanReadableToBytes('2GiB'): self.logger.info('This is a large operation, and could take a while.') def _MakeTempFile(self, file_size=0, mem_metadata=False, mem_data=False, prefix='gsutil_test_file', random_ratio=100): """Creates a temporary file of the given size and returns its path. Args: file_size: The size of the temporary file to create. mem_metadata: If true, store md5 and file size in memory at temp_file_dict[fpath].md5, tempfile_data[fpath].file_size. mem_data: If true, store the file data in memory at temp_file_dict[fpath].data prefix: The prefix to use for the temporary file. Defaults to gsutil_test_file. random_ratio: The percentage of randomly generated data to write. This can be any number between 0 and 100 (inclusive), with 0 producing uniform data, and 100 producing random data. Returns: The file path of the created temporary file. """ fd, fpath = tempfile.mkstemp(suffix='.bin', prefix=prefix, dir=self.directory, text=False) with os.fdopen(fd, 'wb') as fp: _GenerateFileData(fp, file_size, random_ratio) if mem_metadata or mem_data: with open(fpath, 'rb') as fp: file_size = GetFileSize(fp) if mem_metadata else None md5 = CalculateB64EncodedMd5FromContents(fp) if mem_metadata else None data = fp.read() if mem_data else None temp_file_dict[fpath] = FileDataTuple(file_size, md5, data) self.temporary_files.add(fpath) return fpath def _SetUp(self): """Performs setup operations needed before diagnostics can be run.""" # Stores test result data. self.results = {} # Set of file paths for local temporary files. self.temporary_files = set() # Set of names for test objects that exist in the test bucket. self.temporary_objects = set() # Total number of HTTP requests made. self.total_requests = 0 # Total number of HTTP 5xx errors. self.request_errors = 0 # Number of responses, keyed by response code. self.error_responses_by_code = defaultdict(int) # Total number of socket errors. self.connection_breaks = 0 # Boolean to prevent doing cleanup twice. self.teardown_completed = False # Create files for latency test. if self.LAT in self.diag_tests: self.latency_files = [] for file_size in self.test_lat_file_sizes: fpath = self._MakeTempFile(file_size, mem_metadata=True, mem_data=True) self.latency_files.append(fpath) # Create files for throughput tests. if self.diag_tests.intersection( (self.RTHRU, self.WTHRU, self.RTHRU_FILE, self.WTHRU_FILE)): # Create a file for warming up the TCP connection. self.tcp_warmup_file = self._MakeTempFile(5 * 1024 * 1024, mem_metadata=True, mem_data=True) # For in memory tests, throughput tests transfer the same object N times # instead of creating N objects, in order to avoid excessive memory usage. if self.diag_tests.intersection((self.RTHRU, self.WTHRU)): self.mem_thru_file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, mem_data=True, random_ratio=self.gzip_compression_ratio) self.mem_thru_object_name = os.path.basename(self.mem_thru_file_name) # For tests that use disk I/O, it is necessary to create N objects in # in order to properly measure the performance impact of seeks. if self.diag_tests.intersection((self.RTHRU_FILE, self.WTHRU_FILE)): # List of file names and corresponding object names to use for file # throughput tests. self.thru_file_names = [] self.thru_object_names = [] free_disk_space = CheckFreeSpace(self.directory) if free_disk_space >= self.thru_filesize * self.num_objects: self.logger.info( '\nCreating %d local files each of size %s.' % (self.num_objects, MakeHumanReadable(self.thru_filesize))) self._WarnIfLargeData() for _ in range(self.num_objects): file_name = self._MakeTempFile( self.thru_filesize, mem_metadata=True, random_ratio=self.gzip_compression_ratio) self.thru_file_names.append(file_name) self.thru_object_names.append(os.path.basename(file_name)) else: raise CommandException( 'Not enough free disk space for throughput files: ' '%s of disk space required, but only %s available.' % (MakeHumanReadable(self.thru_filesize * self.num_objects), MakeHumanReadable(free_disk_space))) # Dummy file buffer to use for downloading that goes nowhere. self.discard_sink = DummyFile() # Filter out misleading progress callback output and the incorrect # suggestion to use gsutil -m perfdiag. self.logger.addFilter(self._PerfdiagFilter()) def _TearDown(self): """Performs operations to clean things up after performing diagnostics.""" if not self.teardown_completed: temp_file_dict.clear() try: for fpath in self.temporary_files: os.remove(fpath) if self.delete_directory: os.rmdir(self.directory) except OSError: pass # I usually hate leaving commented out code in commits, but this is a # rare exception. This commented out code attempted to clean up the # files created by the perfdiag command. If there is only one process and # thread, it worked by serializing the deletes. If it had more than one # process (multiple threads on a single process seemed to work fine, but # don't quote me on that), it tried to fire up a multi-processing delete # that used the existing process pool created during the previous part of # the perfdiag command execution. This randomly fails in Python 3. So I # have left this code here, commented out, and have replaced it with the # sequential delete part only. That this fails in Python 3 will probably # present as other problems later. At least a multiprocessing rm using # gsutil -m appears to always work, so it is something to do with this # usage of parallel override and existing process pools, I think. # PS - only fails when using prefer XML. # if self.threads > 1 or self.processes > 1: # args = [obj for obj in self.temporary_objects] # self.Apply( # _DeleteWrapper, args, _PerfdiagExceptionHandler, # arg_checker=DummyArgChecker, # parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, # process_count=self.processes, thread_count=self.threads) # else: # for object_name in self.temporary_objects: # self.Delete(object_name, self.gsutil_api) for object_name in self.temporary_objects: self.Delete(object_name, self.gsutil_api) self.teardown_completed = True @contextlib.contextmanager def _Time(self, key, bucket): """A context manager that measures time. A context manager that prints a status message before and after executing the inner command and times how long the inner command takes. Keeps track of the timing, aggregated by the given key. Args: key: The key to insert the timing value into a dictionary bucket. bucket: A dictionary to place the timing value in. Yields: For the context manager. """ self.logger.info('%s starting...', key) t0 = time.time() yield t1 = time.time() bucket[key].append(t1 - t0) self.logger.info('%s done.', key) def _RunOperation(self, func): """Runs an operation with retry logic. Args: func: The function to run. Returns: True if the operation succeeds, False if aborted. """ # We retry on httplib exceptions that can happen if the socket was closed # by the remote party or the connection broke because of network issues. # Only the BotoServerError is counted as a 5xx error towards the retry # limit. success = False server_error_retried = 0 total_retried = 0 i = 0 return_val = None while not success: next_sleep = min(random.random() * (2*i) + 1, GetMaxRetryDelay()) try: return_val = func() self.total_requests += 1 success = True except tuple(self.exceptions) as e: total_retried += 1 if total_retried > self.MAX_TOTAL_RETRIES: self.logger.info('Reached maximum total retries. Not retrying.') break if isinstance(e, ServiceException): if e.status >= 500: self.error_responses_by_code[e.status] += 1 self.total_requests += 1 self.request_errors += 1 server_error_retried += 1 time.sleep(next_sleep) else: raise if server_error_retried > self.MAX_SERVER_ERROR_RETRIES: self.logger.info( 'Reached maximum server error retries. Not retrying.') break else: self.connection_breaks += 1 return return_val def _RunLatencyTests(self): """Runs latency tests.""" # Stores timing information for each category of operation. self.results['latency'] = defaultdict(list) for i in range(self.num_objects): self.logger.info('\nRunning latency iteration %d...', i + 1) for fpath in self.latency_files: file_data = temp_file_dict[fpath] url = self.bucket_url.Clone() url.object_name = os.path.basename(fpath) file_size = file_data.size readable_file_size = MakeHumanReadable(file_size) self.logger.info( "\nFile of size %s located on disk at '%s' being diagnosed in the " "cloud at '%s'.", readable_file_size, fpath, url) upload_target = StorageUrlToUploadObjectMetadata(url) def _Upload(): io_fp = six.BytesIO(file_data.data) with self._Time('UPLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.UploadObject(io_fp, upload_target, size=file_size, provider=self.provider, fields=['name']) self._RunOperation(_Upload) def _Metadata(): with self._Time('METADATA_%d' % file_size, self.results['latency']): return self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, provider=self.provider, fields=['name', 'contentType', 'mediaLink', 'size']) # Download will get the metadata first if we don't pass it in. download_metadata = self._RunOperation(_Metadata) serialization_data = GetDownloadSerializationData(download_metadata) def _Download(): with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']): self.gsutil_api.GetObjectMedia( url.bucket_name, url.object_name, self.discard_sink, provider=self.provider, serialization_data=serialization_data) self._RunOperation(_Download) def _Delete(): with self._Time('DELETE_%d' % file_size, self.results['latency']): self.gsutil_api.DeleteObject(url.bucket_name, url.object_name, provider=self.provider) self._RunOperation(_Delete) class _PerfdiagFilter(logging.Filter): def filter(self, record): # Used to prevent unnecessary output when using multiprocessing. msg = record.getMessage() return not (('Copying file:///' in msg) or ('Copying gs://' in msg) or ('Computing CRC' in msg) or ('gsutil -m perfdiag' in msg)) def _PerfdiagExceptionHandler(self, e): """Simple exception handler to allow post-completion status.""" self.logger.error(str(e)) def PerformFannedDownload(self, need_to_slice, object_names, file_names, serialization_data): """Performs a parallel download of multiple objects using the fan strategy. Args: need_to_slice: If True, additionally apply the slice strategy to each object in object_names. object_names: A list of object names to be downloaded. Each object must already exist in the test bucket. file_names: A list, corresponding by index to object_names, of file names for downloaded data. If None, discard downloaded data. serialization_data: A list, corresponding by index to object_names, of serialization data for each object. """ args = [] for i in range(len(object_names)): file_name = file_names[i] if file_names else None args.append( FanDownloadTuple(need_to_slice, object_names[i], file_name, serialization_data[i])) self.Apply( _DownloadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedDownload(self, object_name, file_name, serialization_data): """Performs a download of an object using the slice strategy. Args: object_name: The name of the object to download. file_name: The name of the file to download data to, or None if data should be discarded. serialization_data: The serialization data for the object. """ if file_name: with open(file_name, 'ab') as fp: fp.truncate(self.thru_filesize) component_size = DivideAndCeil(self.thru_filesize, self.num_slices) args = [] for i in range(self.num_slices): start_byte = i component_size end_byte = min((i + 1) * (component_size) - 1, self.thru_filesize - 1) args.append( SliceDownloadTuple(object_name, file_name, serialization_data, start_byte, end_byte)) self.Apply( _DownloadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformFannedUpload(self, need_to_slice, file_names, object_names, use_file, gzip_encoded=False): """Performs a parallel upload of multiple files using the fan strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data for file_name should be present in temp_file_dict if use_file is specified as False. Args: need_to_slice: If True, additionally apply the slice strategy to each file in file_names. file_names: A list of file names to be uploaded. object_names: A list, corresponding by by index to file_names, of object names to upload data to. use_file: If true, use disk I/O, otherwise read upload data from memory. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. """ args = [] for i in range(len(file_names)): args.append( FanUploadTuple(need_to_slice, file_names[i], object_names[i], use_file, gzip_encoded)) self.Apply( _UploadObject, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def PerformSlicedUpload(self, file_name, object_name, use_file, gsutil_api, gzip_encoded=False): """Performs a parallel upload of a file using the slice strategy. The metadata for file_name should be present in temp_file_dict prior to calling. Also, the data from for file_name should be present in temp_file_dict if use_file is specified as False. Args: file_name: The name of the file to upload. object_name: The name of the object to upload to. use_file: If true, use disk I/O, otherwise read upload data from memory. gsutil_api: CloudApi instance to use for operations in this thread. gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a semaphore is used to limit resource usage. """ # Divide the file into components. component_size = DivideAndCeil(self.thru_filesize, self.num_slices) component_object_names = ([ object_name + str(i) for i in range(self.num_slices) ]) args = [] for i in range(self.num_slices): component_start = i * component_size component_size = min(component_size, temp_file_dict[file_name].size - component_start) args.append( SliceUploadTuple(file_name, component_object_names[i], use_file, component_start, component_size, gzip_encoded)) # Upload the components in parallel. try: self.Apply( _UploadSlice, args, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) # Compose the components into an object. request_components = [] for i in range(self.num_slices): src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=component_object_names[i])) request_components.append(src_obj_metadata) def _Compose(): dst_obj_metadata = apitools_messages.Object() dst_obj_metadata.name = object_name dst_obj_metadata.bucket = self.bucket_url.bucket_name gsutil_api.ComposeObject(request_components, dst_obj_metadata, provider=self.provider) self._RunOperation(_Compose) finally: # Delete the temporary components. self.Apply( _DeleteWrapper, component_object_names, _PerfdiagExceptionHandler, ('total_requests', 'request_errors'), arg_checker=DummyArgChecker, parallel_operations_override=self.ParallelOverrideReason.PERFDIAG, process_count=self.processes, thread_count=self.threads) def _RunReadThruTests(self, use_file=False): """Runs read throughput tests.""" test_name = 'read_throughput_file' if use_file else 'read_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning read throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy } # Copy the file(s) to the test bucket, and also get the serialization data # so that we can pass it to download. if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names serialization_data = [] for i in range(self.num_objects): self.temporary_objects.add(self.thru_object_names[i]) if self.WTHRU_FILE in self.diag_tests: # If we ran the WTHRU_FILE test, then the objects already exist. obj_metadata = self.gsutil_api.GetObjectMetadata( self.bucket_url.bucket_name, self.thru_object_names[i], fields=['size', 'mediaLink'], provider=self.bucket_url.scheme) else: obj_metadata = self.Upload(self.thru_file_names[i], self.thru_object_names[i], self.gsutil_api, use_file) # File overwrite causes performance issues with sliced downloads. # Delete the file and reopen it for download. This matches what a real # download would look like. os.unlink(self.thru_file_names[i]) open(self.thru_file_names[i], 'ab').close() serialization_data.append(GetDownloadSerializationData(obj_metadata)) else: # For in-memory test only use one file but copy it num_objects times, to # allow scalability in num_objects. self.temporary_objects.add(self.mem_thru_object_name) obj_metadata = self.Upload(self.mem_thru_file_name, self.mem_thru_object_name, self.gsutil_api, use_file) file_names = None object_names = [self.mem_thru_object_name] * self.num_objects serialization_data = ([GetDownloadSerializationData(obj_metadata)] * self.num_objects) # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) self.Download(warmup_obj_name, self.gsutil_api) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.Download(object_names[i], self.gsutil_api, file_name, serialization_data[i]) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedDownload(need_to_slice, object_names, file_names, serialization_data) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): file_name = file_names[i] if use_file else None self.PerformSlicedDownload(object_names[i], file_name, serialization_data[i]) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunWriteThruTests(self, use_file=False): """Runs write throughput tests.""" test_name = 'write_throughput_file' if use_file else 'write_throughput' file_io_string = 'with file I/O' if use_file else '' self.logger.info( '\nRunning write throughput tests %s (%s objects of size %s)' % (file_io_string, self.num_objects, MakeHumanReadable( self.thru_filesize))) self._WarnIfLargeData() self.results[test_name] = { 'file_size': self.thru_filesize, 'processes': self.processes, 'threads': self.threads, 'parallelism': self.parallel_strategy, 'gzip_encoded_writes': self.gzip_encoded_writes, 'gzip_compression_ratio': self.gzip_compression_ratio } # Warmup the TCP connection. warmup_obj_name = os.path.basename(self.tcp_warmup_file) self.temporary_objects.add(warmup_obj_name) self.Upload(self.tcp_warmup_file, warmup_obj_name, self.gsutil_api) if use_file: # For test with file I/O use N files on disk to preserve seek performance. file_names = self.thru_file_names object_names = self.thru_object_names else: # For in-memory test only use one file but copy it num_objects times, to # allow for scalability in num_objects. file_names = [self.mem_thru_file_name] * self.num_objects object_names = ([ self.mem_thru_object_name + str(i) for i in range(self.num_objects) ]) for object_name in object_names: self.temporary_objects.add(object_name) t0 = time.time() if self.processes == 1 and self.threads == 1: for i in range(self.num_objects): self.Upload(file_names[i], object_names[i], self.gsutil_api, use_file, gzip_encoded=self.gzip_encoded_writes) else: if self.parallel_strategy in (self.FAN, self.BOTH): need_to_slice = (self.parallel_strategy == self.BOTH) self.PerformFannedUpload(need_to_slice, file_names, object_names, use_file, gzip_encoded=self.gzip_encoded_writes) elif self.parallel_strategy == self.SLICE: for i in range(self.num_objects): self.PerformSlicedUpload(file_names[i], object_names[i], use_file, self.gsutil_api, gzip_encoded=self.gzip_encoded_writes) t1 = time.time() time_took = t1 - t0 total_bytes_copied = self.thru_filesize * self.num_objects bytes_per_second = total_bytes_copied / time_took self.results[test_name]['time_took'] = time_took self.results[test_name]['total_bytes_copied'] = total_bytes_copied self.results[test_name]['bytes_per_second'] = bytes_per_second def _RunListTests(self): """Runs eventual consistency listing latency tests.""" self.results['listing'] = {'num_files': self.num_objects} # Generate N random objects to put into the bucket. list_objects = [] args = [] # Differentiate objects created by each perfdiag execution so that leftovers # from a previous run (if perfdiag could not exit gracefully and delete # them) do not affect this run. random_id = ''.join( [random.choice(string.ascii_lowercase) for _ in range(10)]) list_prefix = 'gsutil-perfdiag-list-' + random_id + '-' for _ in range(self.num_objects): fpath = self._MakeTempFile(0, mem_data=True, mem_metadata=True, prefix=list_prefix) object_name = os.path.basename(fpath) list_objects.append(object_name) args.append(FanUploadTuple(False, fpath, object_name, False, False)) self.temporary_objects.add(object_name) # Add the objects to the bucket. self.logger.info('\nWriting %s objects for listing test...', self.num_objects) self.Apply(_UploadObject, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) list_latencies = [] files_seen = [] total_start_time = time.time() expected_objects = set(list_objects) found_objects = set() def _List(): """Lists and returns objects in the bucket. Also records latency.""" t0 = time.time() objects = list( self.gsutil_api.ListObjects(self.bucket_url.bucket_name, prefix=list_prefix, delimiter='/', provider=self.provider, fields=['items/name'])) if len(objects) > self.num_objects: self.logger.warning( 'Listing produced more than the expected %d object(s).', self.num_objects) t1 = time.time() list_latencies.append(t1 - t0) return set([obj.data.name for obj in objects]) def _ListAfterUpload(): names = _List() found_objects.update(names & expected_objects) files_seen.append(len(found_objects)) self.logger.info('Listing bucket %s waiting for %s objects to appear...', self.bucket_url.bucket_name, self.num_objects) while expected_objects - found_objects: self._RunOperation(_ListAfterUpload) if expected_objects - found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['insert'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } args = [object_name for object_name in list_objects] self.logger.info('Deleting %s objects for listing test...', self.num_objects) self.Apply(_DeleteWrapper, args, _PerfdiagExceptionHandler, arg_checker=DummyArgChecker) self.logger.info('Listing bucket %s waiting for %s objects to disappear...', self.bucket_url.bucket_name, self.num_objects) list_latencies = [] files_seen = [] total_start_time = time.time() found_objects = set(list_objects) while found_objects: def _ListAfterDelete(): names = _List() found_objects.intersection_update(names) files_seen.append(len(found_objects)) self._RunOperation(_ListAfterDelete) if found_objects: if time.time() - total_start_time > self.MAX_LISTING_WAIT_TIME: self.logger.warning('Maximum time reached waiting for listing.') break total_end_time = time.time() self.results['listing']['delete'] = { 'num_listing_calls': len(list_latencies), 'list_latencies': list_latencies, 'files_seen_after_listing': files_seen, 'time_took': total_end_time - total_start_time, } def Upload(self, file_name, object_name, gsutil_api, use_file=False, file_start=0, file_size=None, gzip_encoded=False): """Performs an upload to the test bucket. The file is uploaded to the bucket referred to by self.bucket_url, and has name object_name. Args: file_name: The path to the local file, and the key to its entry in temp_file_dict. object_name: The name of the remote object. gsutil_api: Cloud API instance to use for this upload. use_file: If true, use disk I/O, otherwise read everything from memory. file_start: The first byte in the file to upload to the object. (only should be specified for sliced uploads) file_size: The size of the file to upload. (only should be specified for sliced uploads) gzip_encoded: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. Returns: Uploaded Object Metadata. """ fp = None if file_size is None: file_size = temp_file_dict[file_name].size upload_url = self.bucket_url.Clone() upload_url.object_name = object_name upload_target = StorageUrlToUploadObjectMetadata(upload_url) try: if use_file: fp = FilePart(file_name, file_start, file_size) else: data = temp_file_dict[file_name].data[file_start:file_start + file_size] fp = six.BytesIO(data) def _InnerUpload(): if file_size < ResumableThreshold(): return gsutil_api.UploadObject(fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], gzip_encoded=gzip_encoded) else: return gsutil_api.UploadObjectResumable( fp, upload_target, provider=self.provider, size=file_size, fields=['name', 'mediaLink', 'size'], tracker_callback=_DummyTrackerCallback, gzip_encoded=gzip_encoded) # Because perfdiag does not go through copy_helper, it misses the # resource lock. Here, if the upload is compressed, the lock is applied # manually. def _InnerUploadLock(): with gslib.command.concurrent_compressed_upload_lock: return _InnerUpload() upload_func = _InnerUploadLock if gzip_encoded else _InnerUpload return self._RunOperation(upload_func) finally: if fp: fp.close() def Download(self, object_name, gsutil_api, file_name=None, serialization_data=None, start_byte=0, end_byte=None): """Downloads an object from the test bucket. Args: object_name: The name of the object (in the test bucket) to download. gsutil_api: CloudApi instance to use for the download. file_name: Optional file name to write downloaded data to. If None, downloaded data is discarded immediately. serialization_data: Optional serialization data, used so that we don't have to get the metadata before downloading. start_byte: The first byte in the object to download. (only should be specified for sliced downloads) end_byte: The last byte in the object to download. (only should be specified for sliced downloads) """ fp = None try: if file_name is not None: fp = open(file_name, 'r+b') fp.seek(start_byte) else: fp = self.discard_sink def _InnerDownload(): gsutil_api.GetObjectMedia(self.bucket_url.bucket_name, object_name, fp, provider=self.provider, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data) self._RunOperation(_InnerDownload) finally: if fp: fp.close() def Delete(self, object_name, gsutil_api): """Deletes an object from the test bucket. Args: object_name: The name of the object to delete. gsutil_api: Cloud API instance to use for this delete. """ try: def _InnerDelete(): gsutil_api.DeleteObject(self.bucket_url.bucket_name, object_name, provider=self.provider) self._RunOperation(_InnerDelete) except NotFoundException: pass def _GetDiskCounters(self): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = [] with open('/proc/partitions', 'r') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) retdict = {} with open('/proc/diskstats', 'r') as f: for line in f: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) return retdict def _GetTcpStats(self): """Tries to parse out TCP packet information from netstat output. Returns: A dictionary containing TCP information, or None if netstat is not available. """ # netstat return code is non-zero for -s on Linux, so don't raise on error. try: netstat_output = self._Exec(['netstat', '-s'], return_output=True, raise_on_error=False) except OSError: self.logger.warning('netstat not found on your system; some measurement ' 'data will be missing') return None netstat_output = netstat_output.strip().lower() found_tcp = False tcp_retransmit = None tcp_received = None tcp_sent = None for line in netstat_output.split('\n'): # Header for TCP section is "Tcp:" in Linux/Mac and # "TCP Statistics for" in Windows. if 'tcp:' in line or 'tcp statistics' in line: found_tcp = True # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts" # Windows == "segments retransmitted". if (found_tcp and tcp_retransmit is None and ('segments retransmited' in line or 'retransmit timeouts' in line or 'segments retransmitted' in line)): tcp_retransmit = ''.join(c for c in line if c in string.digits) # Linux+Windows == "segments received", Mac == "packets received". if (found_tcp and tcp_received is None and ('segments received' in line or 'packets received' in line)): tcp_received = ''.join(c for c in line if c in string.digits) # Linux == "segments send out" (sic), Mac+Windows == "packets sent". if (found_tcp and tcp_sent is None and ('segments send out' in line or 'packets sent' in line or 'segments sent' in line)): tcp_sent = ''.join(c for c in line if c in string.digits) result = {} try: result['tcp_retransmit'] = int(tcp_retransmit) result['tcp_received'] = int(tcp_received) result['tcp_sent'] = int(tcp_sent) except (ValueError, TypeError): result['tcp_retransmit'] = None result['tcp_received'] = None result['tcp_sent'] = None return result def _CollectSysInfo(self): """Collects system information.""" sysinfo = {} # All exceptions that might be raised from socket module calls. socket_errors = (socket.error, socket.herror, socket.gaierror, socket.timeout) # Find out whether HTTPS is enabled in Boto. sysinfo['boto_https_enabled'] = boto.config.get('Boto', 'is_secure', True) # Look up proxy info. proxy_host = boto.config.get('Boto', 'proxy', None) proxy_port = boto.config.getint('Boto', 'proxy_port', 0) sysinfo['using_proxy'] = bool(proxy_host) if boto.config.get('Boto', 'proxy_rdns', True if proxy_host else False): self.logger.info( 'DNS lookups are disallowed in this environment, so some information ' 'is not included in this perfdiag run. To allow local DNS lookups ' 'while using a proxy, set proxy_rdns to False in your boto file.') # Get the local IP address from socket lib. try: sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname()) except socket_errors: sysinfo['ip_address'] = '' # Record the temporary directory used since it can affect performance, e.g. # when on a networked filesystem. sysinfo['tempdir'] = self.directory # Produces an RFC 2822 compliant GMT timestamp. sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000', time.gmtime()) # Execute a CNAME lookup on Google DNS to find what Google server # it's routing to. cmd = ['nslookup', '-type=CNAME', self.XML_API_HOST] try: nslookup_cname_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output) sysinfo['googserv_route'] = m.group('googserv') if m else None except (CommandException, OSError): sysinfo['googserv_route'] = '' # Check the BIOS product name, to determine if we're running on a GCE # instance. # TODO: Add support for checking if we're on GCE from Windows, via WMI. if IS_LINUX: try: with open('/sys/class/dmi/id/product_name', 'r') as fp: sysinfo['on_gce'] = (fp.readline() == 'Google Compute Engine\n') except OSError: pass # If we're running on GCE include details about the instance. if sysinfo.get('on_gce', False): hostname = socket.gethostname() cmd = ['gcloud', 'compute', 'instances', 'list', '--filter=', hostname] try: # "gcloud compute" commands will (hopefully) fail on these envs due to # lack of credentials/permissions to access compute resources. mute_stderr = IsRunningInCiEnvironment() sysinfo['gce_instance_info'] = (self._Exec(cmd, return_output=True, mute_stderr=mute_stderr)) except (CommandException, OSError): sysinfo['gce_instance_info'] = '' # Record info about location and storage class of the bucket being used for # performance testing. bucket_info = self.gsutil_api.GetBucket(self.bucket_url.bucket_name, fields=['location', 'storageClass'], provider=self.bucket_url.scheme) sysinfo['bucket_location'] = bucket_info.location sysinfo['bucket_storageClass'] = bucket_info.storageClass # Try to determine the latency of a DNS lookup for the Google hostname # endpoint. Note: we don't piggyback on gethostbyname_ex below because # the _ex version requires an extra RTT. try: t0 = time.time() socket.gethostbyname(self.XML_API_HOST) t1 = time.time() sysinfo['google_host_dns_latency'] = t1 - t0 except socket_errors: pass # Look up IP addresses for Google Server. try: (hostname, _, ipaddrlist) = socket.gethostbyname_ex(self.XML_API_HOST) sysinfo['googserv_ips'] = ipaddrlist except socket_errors: ipaddrlist = [] sysinfo['googserv_ips'] = [] # Reverse lookup the hostnames for the Google Server IPs. sysinfo['googserv_hostnames'] = [] for googserv_ip in ipaddrlist: try: (hostname, _, ipaddrlist) = socket.gethostbyaddr(googserv_ip) sysinfo['googserv_hostnames'].append(hostname) except socket_errors: pass # Query o-o to find out what the Google DNS thinks is the user's IP. try: cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.'] nslookup_txt_output = self._Exec(cmd, return_output=True, mute_stderr=True) m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output) sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None except (CommandException, OSError): sysinfo['dns_o-o_ip'] = '' # Try to determine the latency of connecting to the Google hostname # endpoint. sysinfo['google_host_connect_latencies'] = {} for googserv_ip in ipaddrlist: try: sock = socket.socket() t0 = time.time() sock.connect((googserv_ip, self.XML_API_PORT)) t1 = time.time() sysinfo['google_host_connect_latencies'][googserv_ip] = t1 - t0 except socket_errors: pass # If using a proxy, try to determine the latency of a DNS lookup to resolve # the proxy hostname and the latency of connecting to the proxy. if proxy_host: proxy_ip = None try: t0 = time.time() proxy_ip = socket.gethostbyname(proxy_host) t1 = time.time() sysinfo['proxy_dns_latency'] = t1 - t0 except socket_errors: pass try: sock = socket.socket() t0 = time.time() sock.connect((proxy_ip or proxy_host, proxy_port)) t1 = time.time() sysinfo['proxy_host_connect_latency'] = t1 - t0 except socket_errors: pass # Try and find the number of CPUs in the system if available. try: sysinfo['cpu_count'] = multiprocessing.cpu_count() except NotImplementedError: sysinfo['cpu_count'] = None # For nix platforms, obtain the CPU load. try: sysinfo['load_avg'] = list(os.getloadavg()) except (AttributeError, OSError): sysinfo['load_avg'] = None # Try and collect memory information from /proc/meminfo if possible. mem_total = None mem_free = None mem_buffers = None mem_cached = None try: with open('/proc/meminfo', 'r') as f: for line in f: if line.startswith('MemTotal'): mem_total = (int(''.join(c for c in line if c in string.digits)) 1000) elif line.startswith('MemFree'): mem_free = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Buffers'): mem_buffers = (int(''.join(c for c in line if c in string.digits)) * 1000) elif line.startswith('Cached'): mem_cached = (int(''.join(c for c in line if c in string.digits)) * 1000) except (IOError, ValueError): pass sysinfo['meminfo'] = { 'mem_total': mem_total, 'mem_free': mem_free, 'mem_buffers': mem_buffers, 'mem_cached': mem_cached } # Get configuration attributes from config module. sysinfo['gsutil_config'] = {} for attr in dir(config): attr_value = getattr(config, attr) # Filter out multiline strings that are not useful. if attr.isupper() and not (isinstance(attr_value, six.string_types) and '\n' in attr_value): sysinfo['gsutil_config'][attr] = attr_value sysinfo['tcp_proc_values'] = {} stats_to_check = [ '/proc/sys/net/core/rmem_default', '/proc/sys/net/core/rmem_max', '/proc/sys/net/core/wmem_default', '/proc/sys/net/core/wmem_max', '/proc/sys/net/ipv4/tcp_timestamps', '/proc/sys/net/ipv4/tcp_sack', '/proc/sys/net/ipv4/tcp_window_scaling', ] for fname in stats_to_check: try: with open(fname, 'r') as f: value = f.read() sysinfo['tcp_proc_values'][os.path.basename(fname)] = value.strip() except IOError: pass self.results['sysinfo'] = sysinfo def _DisplayStats(self, trials): """Prints out mean, standard deviation, median, and 90th percentile.""" n = len(trials) mean = float(sum(trials)) / n stdev = math.sqrt(sum((x - mean)*2 for x in trials) / n) text_util.print_to_fd(str(n).rjust(6), '', end=' ') text_util.print_to_fd(('%.1f' % (mean 1000)).rjust(9), '', end=' ') text_util.print_to_fd(('%.1f' % (stdev * 1000)).rjust(12), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '', end=' ') text_util.print_to_fd(('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), '') def _DisplayResults(self): """Displays results collected from diagnostic run.""" text_util.print_to_fd() text_util.print_to_fd('=' * 78) text_util.print_to_fd('DIAGNOSTIC RESULTS'.center(78)) text_util.print_to_fd('=' * 78) if 'latency' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Latency'.center(78)) text_util.print_to_fd('-' * 78) text_util.print_to_fd( 'Operation Size Trials Mean (ms) Std Dev (ms) ' 'Median (ms) 90th % (ms)') text_util.print_to_fd( '========= ========= ====== ========= ============ ' '=========== ===========') for key in sorted(self.results['latency']): trials = sorted(self.results['latency'][key]) op, numbytes = key.split('_') numbytes = int(numbytes) if op == 'METADATA': text_util.print_to_fd('Metadata'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DOWNLOAD': text_util.print_to_fd('Download'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'UPLOAD': text_util.print_to_fd('Upload'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if op == 'DELETE': text_util.print_to_fd('Delete'.rjust(9), '', end=' ') text_util.print_to_fd(MakeHumanReadable(numbytes).rjust(9), '', end=' ') self._DisplayStats(trials) if 'write_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput'.center(78)) text_util.print_to_fd('-' * 78) write_thru = self.results['write_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru['file_size']), MakeHumanReadable(write_thru['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))) if 'parallelism' in write_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru['parallelism']) if 'write_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Write Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) write_thru_file = self.results['write_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(write_thru_file['file_size']), MakeHumanReadable(write_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Write throughput: %s/s.' % (MakeBitsHumanReadable(write_thru_file['bytes_per_second'] * 8))) if 'parallelism' in write_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % write_thru_file['parallelism']) if 'read_throughput' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput'.center(78)) text_util.print_to_fd('-' * 78) read_thru = self.results['read_throughput'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru['file_size']), MakeHumanReadable(read_thru['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))) if 'parallelism' in read_thru: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru['parallelism']) if 'read_throughput_file' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Read Throughput With File I/O'.center(78)) text_util.print_to_fd('-' * 78) read_thru_file = self.results['read_throughput_file'] text_util.print_to_fd( 'Copied %s %s file(s) for a total transfer size of %s.' % (self.num_objects, MakeHumanReadable(read_thru_file['file_size']), MakeHumanReadable(read_thru_file['total_bytes_copied']))) text_util.print_to_fd( 'Read throughput: %s/s.' % (MakeBitsHumanReadable(read_thru_file['bytes_per_second'] * 8))) if 'parallelism' in read_thru_file: # Compatibility with old versions. text_util.print_to_fd('Parallelism strategy: %s' % read_thru_file['parallelism']) if 'listing' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('Listing'.center(78)) text_util.print_to_fd('-' * 78) listing = self.results['listing'] insert = listing['insert'] delete = listing['delete'] text_util.print_to_fd('After inserting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % insert['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % insert['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in insert['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, insert['files_seen_after_listing'])))) text_util.print_to_fd('After deleting %s objects:' % listing['num_files']) text_util.print_to_fd( (' Total time for objects to appear: %.2g seconds' % delete['time_took'])) text_util.print_to_fd(' Number of listing calls made: %s' % delete['num_listing_calls']) text_util.print_to_fd( (' Individual listing call latencies: [%s]' % ', '.join('%.2gs' % lat for lat in delete['list_latencies']))) text_util.print_to_fd( (' Files reflected after each call: [%s]' % ', '.join(map(str, delete['files_seen_after_listing'])))) if 'sysinfo' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('System Information'.center(78)) text_util.print_to_fd('-' * 78) info = self.results['sysinfo'] text_util.print_to_fd('IP Address: \n %s' % info['ip_address']) text_util.print_to_fd('Temporary Directory: \n %s' % info['tempdir']) text_util.print_to_fd('Bucket URI: \n %s' % self.results['bucket_uri']) text_util.print_to_fd('gsutil Version: \n %s' % self.results.get('gsutil_version', 'Unknown')) text_util.print_to_fd('boto Version: \n %s' % self.results.get('boto_version', 'Unknown')) if 'gmt_timestamp' in info: ts_string = info['gmt_timestamp'] timetuple = None try: # Convert RFC 2822 string to Linux timestamp. timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000') except ValueError: pass if timetuple: # Converts the GMT time tuple to local Linux timestamp. localtime = calendar.timegm(timetuple) localdt = datetime.datetime.fromtimestamp(localtime) text_util.print_to_fd('Measurement time: \n %s' % localdt.strftime('%Y-%m-%d %I:%M:%S %p %Z')) if 'on_gce' in info: text_util.print_to_fd('Running on GCE: \n %s' % info['on_gce']) if info['on_gce']: text_util.print_to_fd('GCE Instance:\n\t%s' % info['gce_instance_info'].replace('\n', '\n\t')) text_util.print_to_fd('Bucket location: \n %s' % info['bucket_location']) text_util.print_to_fd('Bucket storage class: \n %s' % info['bucket_storageClass']) text_util.print_to_fd('Google Server: \n %s' % info['googserv_route']) text_util.print_to_fd('Google Server IP Addresses: \n %s' % ('\n '.join(info['googserv_ips']))) text_util.print_to_fd('Google Server Hostnames: \n %s' % ('\n '.join(info['googserv_hostnames']))) text_util.print_to_fd('Google DNS thinks your IP is: \n %s' % info['dns_o-o_ip']) text_util.print_to_fd('CPU Count: \n %s' % info['cpu_count']) text_util.print_to_fd('CPU Load Average: \n %s' % info['load_avg']) try: text_util.print_to_fd(('Total Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_total']))) # Free memory is really MemFree + Buffers + Cached. text_util.print_to_fd('Free Memory: \n %s' % MakeHumanReadable(info['meminfo']['mem_free'] + info['meminfo']['mem_buffers'] + info['meminfo']['mem_cached'])) except TypeError: pass if 'netstat_end' in info and 'netstat_start' in info: netstat_after = info['netstat_end'] netstat_before = info['netstat_start'] for tcp_type in ('sent', 'received', 'retransmit'): try: delta = (netstat_after['tcp_%s' % tcp_type] - netstat_before['tcp_%s' % tcp_type]) text_util.print_to_fd('TCP segments %s during test:\n %d' % (tcp_type, delta)) except TypeError: pass else: text_util.print_to_fd( 'TCP segment counts not available because "netstat" was not ' 'found during test runs') if 'disk_counters_end' in info and 'disk_counters_start' in info: text_util.print_to_fd('Disk Counter Deltas:\n', end=' ') disk_after = info['disk_counters_end'] disk_before = info['disk_counters_start'] text_util.print_to_fd('', 'disk'.rjust(6), end=' ') for colname in [ 'reads', 'writes', 'rbytes', 'wbytes', 'rtime', 'wtime' ]: text_util.print_to_fd(colname.rjust(8), end=' ') text_util.print_to_fd() for diskname in sorted(disk_after): before = disk_before[diskname] after = disk_after[diskname] (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after text_util.print_to_fd('', diskname.rjust(6), end=' ') deltas = [ reads2 - reads1, writes2 - writes1, rbytes2 - rbytes1, wbytes2 - wbytes1, rtime2 - rtime1, wtime2 - wtime1 ] for delta in deltas: text_util.print_to_fd(str(delta).rjust(8), end=' ') text_util.print_to_fd() if 'tcp_proc_values' in info: text_util.print_to_fd('TCP /proc values:\n', end=' ') for item in six.iteritems(info['tcp_proc_values']): text_util.print_to_fd(' %s = %s' % item) if 'boto_https_enabled' in info: text_util.print_to_fd('Boto HTTPS Enabled: \n %s' % info['boto_https_enabled']) if 'using_proxy' in info: text_util.print_to_fd('Requests routed through proxy: \n %s' % info['using_proxy']) if 'google_host_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for Google Storage server (ms): ' '\n %.1f' % (info['google_host_dns_latency'] * 1000.0))) if 'google_host_connect_latencies' in info: text_util.print_to_fd( 'Latencies connecting to Google Storage server IPs (ms):') for ip, latency in six.iteritems(info['google_host_connect_latencies']): text_util.print_to_fd(' %s = %.1f' % (ip, latency * 1000.0)) if 'proxy_dns_latency' in info: text_util.print_to_fd( ('Latency of the DNS lookup for the configured proxy (ms): ' '\n %.1f' % (info['proxy_dns_latency'] * 1000.0))) if 'proxy_host_connect_latency' in info: text_util.print_to_fd( ('Latency connecting to the configured proxy (ms): \n %.1f' % (info['proxy_host_connect_latency'] * 1000.0))) if 'request_errors' in self.results and 'total_requests' in self.results: text_util.print_to_fd() text_util.print_to_fd('-' * 78) text_util.print_to_fd('In-Process HTTP Statistics'.center(78)) text_util.print_to_fd('-' * 78) total = int(self.results['total_requests']) numerrors = int(self.results['request_errors']) numbreaks = int(self.results['connection_breaks']) availability = (((total - numerrors) / float(total)) * 100 if total > 0 else 100) text_util.print_to_fd('Total HTTP requests made: %d' % total) text_util.print_to_fd('HTTP 5xx errors: %d' % numerrors) text_util.print_to_fd('HTTP connections broken: %d' % numbreaks) text_util.print_to_fd('Availability: %.7g%%' % availability) if 'error_responses_by_code' in self.results: sorted_codes = sorted( six.iteritems(self.results['error_responses_by_code'])) if sorted_codes: text_util.print_to_fd('Error responses by code:') text_util.print_to_fd('\n'.join(' %s: %s' % c for c in sorted_codes)) if self.output_file: with open(self.output_file, 'w') as f: json.dump(self.results, f, indent=2) text_util.print_to_fd() text_util.print_to_fd("Output file written to '%s'." % self.output_file) text_util.print_to_fd() def _ParsePositiveInteger(self, val, msg): """Tries to convert val argument to a positive integer. Args: val: The value (as a string) to convert to a positive integer. msg: The error message to place in the CommandException on an error. Returns: A valid positive integer. Raises: CommandException: If the supplied value is not a valid positive integer. """ try: val = int(val) if val < 1: raise CommandException(msg) return val except ValueError: raise CommandException(msg) def _ParseArgs(self): """Parses arguments for perfdiag command.""" # From -n. self.num_objects = 5 # From -c. self.processes = 1 # From -k. self.threads = 1 # From -p self.parallel_strategy = None # From -y self.num_slices = 4 # From -s. self.thru_filesize = 1048576 # From -d. self.directory = tempfile.gettempdir() # Keep track of whether or not to delete the directory upon completion. self.delete_directory = False # From -t. self.diag_tests = set(self.DEFAULT_DIAG_TESTS) # From -o. self.output_file = None # From -i. self.input_file = None # From -m. self.metadata_keys = {} # From -j. self.gzip_encoded_writes = False self.gzip_compression_ratio = 100 if self.sub_opts: for o, a in self.sub_opts: if o == '-n': self.num_objects = self._ParsePositiveInteger( a, 'The -n parameter must be a positive integer.') if o == '-c': self.processes = self._ParsePositiveInteger( a, 'The -c parameter must be a positive integer.') if o == '-k': self.threads = self._ParsePositiveInteger( a, 'The -k parameter must be a positive integer.') if o == '-p': if a.lower() in self.PARALLEL_STRATEGIES: self.parallel_strategy = a.lower() else: raise CommandException("'%s' is not a valid parallelism strategy." % a) if o == '-y': self.num_slices = self._ParsePositiveInteger( a, 'The -y parameter must be a positive integer.') if o == '-s': try: self.thru_filesize = HumanReadableToBytes(a) except ValueError: raise CommandException('Invalid -s parameter.') if o == '-d': self.directory = a if not os.path.exists(self.directory): self.delete_directory = True os.makedirs(self.directory) if o == '-t': self.diag_tests = set() for test_name in a.strip().split(','): if test_name.lower() not in self.ALL_DIAG_TESTS: raise CommandException("List of test names (-t) contains invalid " "test name '%s'." % test_name) self.diag_tests.add(test_name) if o == '-m': pieces = a.split(':') if len(pieces) != 2: raise CommandException( "Invalid metadata key-value combination '%s'." % a) key, value = pieces self.metadata_keys[key] = value if o == '-o': self.output_file = os.path.abspath(a) if o == '-i': self.input_file = os.path.abspath(a) if not os.path.isfile(self.input_file): raise CommandException("Invalid input file (-i): '%s'." % a) try: with open(self.input_file, 'r') as f: self.results = json.load(f) self.logger.info("Read input file: '%s'.", self.input_file) except ValueError: raise CommandException("Could not decode input file (-i): '%s'." % a) return if o == '-j': self.gzip_encoded_writes = True try: self.gzip_compression_ratio = int(a) except ValueError: self.gzip_compression_ratio = -1 if (self.gzip_compression_ratio < 0 or self.gzip_compression_ratio > 100): raise CommandException( 'The -j parameter must be between 0 and 100 (inclusive).') # If parallelism is specified, default parallelism strategy to fan. if (self.processes > 1 or self.threads > 1) and not self.parallel_strategy: self.parallel_strategy = self.FAN elif self.processes == 1 and self.threads == 1 and self.parallel_strategy: raise CommandException( 'Cannot specify parallelism strategy (-p) without also specifying ' 'multiple threads and/or processes (-c and/or -k).') if not self.args: self.RaiseWrongNumberOfArgumentsException() self.bucket_url = StorageUrlFromString(self.args[0]) self.provider = self.bucket_url.scheme if not self.bucket_url.IsCloudUrl() and self.bucket_url.IsBucket(): raise CommandException('The perfdiag command requires a URL that ' 'specifies a bucket.\n"%s" is not ' 'valid.' % self.args[0]) if (self.thru_filesize > HumanReadableToBytes('2GiB') and (self.RTHRU in self.diag_tests or self.WTHRU in self.diag_tests)): raise CommandException( 'For in-memory tests maximum file size is 2GiB. For larger file ' 'sizes, specify rthru_file and/or wthru_file with the -t option.') perform_slice = self.parallel_strategy in (self.SLICE, self.BOTH) slice_not_available = (self.provider == 's3' and self.diag_tests.intersection(self.WTHRU, self.WTHRU_FILE)) if perform_slice and slice_not_available: raise CommandException('Sliced uploads are not available for s3. ' 'Use -p fan or sequential uploads for s3.') # Ensure the bucket exists. self.gsutil_api.GetBucket(self.bucket_url.bucket_name, provider=self.bucket_url.scheme, fields=['id']) self.exceptions = [ http_client.HTTPException, socket.error, socket.gaierror, socket.timeout, http_client.BadStatusLine, ServiceException ] # Command entry point. def RunCommand(self): """Called by gsutil when the command is being invoked.""" self._ParseArgs() if self.input_file: self._DisplayResults() return 0 # We turn off retries in the underlying boto library because the # _RunOperation function handles errors manually so it can count them. boto.config.set('Boto', 'num_retries', '0') self.logger.info( 'Number of iterations to run: %d\n' 'Base bucket URI: %s\n' 'Number of processes: %d\n' 'Number of threads: %d\n' 'Parallelism strategy: %s\n' 'Throughput file size: %s\n' 'Diagnostics to run: %s\n' 'Gzip compression ratio: %s\n' 'Gzip transport encoding writes: %s', self.num_objects, self.bucket_url, self.processes, self.threads, self.parallel_strategy, MakeHumanReadable(self.thru_filesize), (', '.join(self.diag_tests)), self.gzip_compression_ratio, self.gzip_encoded_writes) try: self._SetUp() # Collect generic system info. self._CollectSysInfo() # Collect netstat info and disk counters before tests (and again later). netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_start'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_start'] = GetDiskCounters() # Record bucket URL. self.results['bucket_uri'] = str(self.bucket_url) self.results['json_format'] = 'perfdiag' self.results['metadata'] = self.metadata_keys if self.LAT in self.diag_tests: self._RunLatencyTests() if self.RTHRU in self.diag_tests: self._RunReadThruTests() # Run WTHRU_FILE before RTHRU_FILE. If data is created in WTHRU_FILE it # will be used in RTHRU_FILE to save time and bandwidth. if self.WTHRU_FILE in self.diag_tests: self._RunWriteThruTests(use_file=True) if self.RTHRU_FILE in self.diag_tests: self._RunReadThruTests(use_file=True) if self.WTHRU in self.diag_tests: self._RunWriteThruTests() if self.LIST in self.diag_tests: self._RunListTests() # Collect netstat info and disk counters after tests. netstat_output = self._GetTcpStats() if netstat_output: self.results['sysinfo']['netstat_end'] = netstat_output if IS_LINUX: self.results['sysinfo']['disk_counters_end'] = GetDiskCounters() self.results['total_requests'] = self.total_requests self.results['request_errors'] = self.request_errors self.results['error_responses_by_code'] = self.error_responses_by_code self.results['connection_breaks'] = self.connection_breaks self.results['gsutil_version'] = gslib.VERSION self.results['boto_version'] = boto.__version__ self._TearDown() self._DisplayResults() finally: # TODO: Install signal handlers so this is performed in response to a # terminating signal; consider multi-threaded object deletes during # cleanup so it happens quickly. self._TearDown() return 0 def StorageUrlToUploadObjectMetadata(storage_url): if storage_url.IsCloudUrl() and storage_url.IsObject(): upload_target = apitools_messages.Object() upload_target.name = storage_url.object_name upload_target.bucket = storage_url.bucket_name return upload_target else: raise CommandException('Non-cloud URL upload target %s was created in ' 'perfdiag implemenation.' % storage_url)	93.4 KB	2025-06-25 08:19
📄 rb.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rb command for deleting cloud storage buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.cloud_api import NotEmptyException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil rb [-f] gs://<bucket_name>... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Delete one or more buckets. Buckets must be empty before you can delete them. Be certain you want to delete a bucket before you do so, as once it is deleted the name becomes available and another user may create a bucket with that name. (But see also "DOMAIN NAMED BUCKETS" under "gsutil help naming" for help carving out parts of the bucket name space.) <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing buckets. If some buckets couldn't be removed, gsutil's exit status will be non-zero even if this flag is set. If no buckets could be removed, the command raises a "no matches" error. """) class RbCommand(Command): """Implementation of gsutil rb command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rb', command_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='f', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rb', help_name_aliases=[ 'deletebucket', 'removebucket', 'removebuckets', 'rmdir', ], help_type='command_help', help_one_line_summary='Remove buckets', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'buckets', 'delete'], flag_map={ '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rb command.""" self.continue_on_error = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': self.continue_on_error = True did_some_work = False some_failed = False for url_str in self.args: wildcard_url = StorageUrlFromString(url_str) if wildcard_url.IsObject(): raise CommandException('"rb" command requires a provider or ' 'bucket URL') # Wrap WildcardIterator call in try/except so we can avoid printing errors # with -f option if a non-existent URL listed, permission denial happens # while listing, etc. try: # Materialize iterator results into a list to catch exceptions. # Since this is listing buckets this list shouldn't be too large to fit # in memory at once. # Also, avoid listing all fields to avoid performing unnecessary bucket # metadata GETs. These would also be problematic when billing is # disabled, as deletes are allowed but GetBucket is not. blrs = list( self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id'])) except: # pylint: disable=bare-except some_failed = True if self.continue_on_error: continue else: raise for blr in blrs: url = blr.storage_url self.logger.info('Removing %s...', url) try: self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) except NotEmptyException as e: some_failed = True if self.continue_on_error: continue elif 'VersionedBucketNotEmpty' in e.reason: raise CommandException('Bucket is not empty. Note: this is a ' 'versioned bucket, so to delete all ' 'objects\nyou need to use:' '\n\tgsutil rm -r %s' % url) else: raise except: # pylint: disable=bare-except some_failed = True if not self.continue_on_error: raise did_some_work = True if not did_some_work: raise CommandException(NO_URLS_MATCHED_TARGET % list(self.args)) return 1 if some_failed else 0	5.71 KB	2025-06-25 08:19
📄 requesterpays.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of requesterpays configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil requesterpays set (on\|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil requesterpays get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable Requester Pays for the specified bucket. """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the Requester Pays configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Requester Pays <https://cloud.google.com/storage/docs/requester-pays>`_ feature enables you to configure a Google Cloud Storage bucket so that the requester pays all costs related to accessing the bucket and its objects. The gsutil requesterpays command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=": "](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://"),' + 'requester_pays.yesno("Enabled", "Disabled"))') class RequesterPaysCommand(Command): """Implementation of gsutil requesterpays command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'requesterpays', usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ # ApiSelector.XML, # TODO: Uncomment once boto changes are added. ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),] }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='requesterpays', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Enable or disable requester pays for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--requester-pays' ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-requester-pays', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetRequesterPays(self): """Gets requesterpays configuration for a bucket.""" requesterpays_arg = self.args[0].lower() if requesterpays_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on\|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( billing=apitools_messages.Bucket.BillingValue()) if requesterpays_arg == 'on': self.logger.info('Enabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = True else: self.logger.info('Disabling requester pays for %s...', url) bucket_metadata.billing.requesterPays = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetRequesterPays(self): """Gets requesterpays configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the requesterpays # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['billing']) for blr in bucket_iter: some_matched = True if blr.root_object.billing and blr.root_object.billing.requesterPays: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Disabled' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the requesterpays command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetRequesterPays metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetRequesterPays requesterpays_arg = self.args[0].lower() if requesterpays_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, requesterpays_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0	8.28 KB	2025-06-25 08:19
📄 retention.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Retention Policy configuration command for buckets.""" from __future__ import absolute_import import time from apitools.base.py import encoding from gslib import metrics from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.retention_util import ConfirmLockRequest from gslib.utils.retention_util import ReleaseEventHoldFuncWrapper from gslib.utils.retention_util import ReleaseTempHoldFuncWrapper from gslib.utils.retention_util import RetentionInSeconds from gslib.utils.retention_util import RetentionPolicyToString from gslib.utils.retention_util import SetEventHoldFuncWrapper from gslib.utils.retention_util import SetTempHoldFuncWrapper from gslib.utils.retention_util import UpdateObjectMetadataExceptionHandler from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.translation_helper import PreconditionsFromHeaders _SET_SYNOPSIS = """ gsutil retention set <retention_period> gs://<bucket_name>... """ _CLEAR_SYNOPSIS = """ gsutil retention clear gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil retention get gs://<bucket_name>... """ _LOCK_SYNOPSIS = """ gsutil retention lock gs://<bucket_name>... """ _EVENT_DEFAULT_SYNOPSIS = """ gsutil retention event-default (set\|release) gs://<bucket_name>... """ _EVENT_SYNOPSIS = """ gsutil retention event (set\|release) gs://<bucket_name>/<object_name>... """ _TEMP_SYNOPSIS = """ gsutil retention temp (set\|release) gs://<bucket_name>/<object_name>... """ _SET_DESCRIPTION = """ <B>SET</B> You can configure a data retention policy for a Cloud Storage bucket that governs how long objects in the bucket must be retained. You can also lock the data retention policy, permanently preventing the policy from being reduced or removed. For more information, see `Retention policies and Bucket Lock <https://cloud.google.com/storage/docs/bucket-lock>`_. The ``gsutil retention set`` command allows you to set or update the retention policy on one or more buckets. To remove an unlocked retention policy from one or more buckets, use the ``gsutil retention clear`` command. The ``set`` sub-command can set a retention policy with the following formats: <B>SET FORMATS</B> Formats for the ``set`` subcommand include: <number>s Specifies retention period of <number> seconds for objects in this bucket. <number>d Specifies retention period of <number> days for objects in this bucket. <number>m Specifies retention period of <number> months for objects in this bucket. <number>y Specifies retention period of <number> years for objects in this bucket. GCS JSON API accepts retention periods as number of seconds. Durations provided in terms of days, months or years are converted to their rough equivalent values in seconds, using the following conversions: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Retention periods must be greater than 0 and less than 100 years. Retention durations must be in only one form (seconds, days, months, or years), and not a combination of them. Note that while it is possible to specify retention durations shorter than a day (using seconds), enforcement of such retention periods is not guaranteed. Such durations may only be used for testing purposes. <B>EXAMPLES</B> Setting a retention policy with a duration of 1 year on a bucket: gsutil retention set 1y gs://my-bucket Setting a retention policy with a duration of 36 months on a bucket: gsutil retention set 36m gs://some-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the retention policy on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention set 1y gs://my-bucket """ _CLEAR_DESCRIPTION = """ <B>CLEAR</B> The ``gsutil retention clear`` command removes an unlocked retention policy from one or more buckets. You cannot remove or reduce the duration of a locked retention policy. <B>EXAMPLES</B> Clearing an unlocked retention policy from a bucket: gsutil retention clear gs://my-bucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil retention get`` command retrieves the retention policy for a given bucket and displays a human-readable representation of the configuration. """ _LOCK_DESCRIPTION = """ <B>LOCK</B> The ``gsutil retention lock`` command PERMANENTLY locks an unlocked retention policy on one or more buckets. CAUTION: A locked retention policy cannot be removed from a bucket or reduced in duration. Once locked, deleting the bucket is the only way to "remove" a retention policy. """ _EVENT_DEFAULT_DESCRIPTION = """ <B>EVENT-DEFAULT</B> The ``gsutil retention event-default`` command sets the default value for an event-based hold on one or more buckets. By setting the default event-based hold on a bucket, newly-created objects inherit that value as their event-based hold (it is not applied retroactively). <B>EXAMPLES</B> Setting the default event-based hold on a bucket: gsutil retention event-default set gs://my-bucket Releasing the default event-based hold on a bucket: gsutil retention event-default release gs://my-bucket You can also provide a precondition on a bucket's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks a bucket's metageneration before setting the default event-based hold on the bucket: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event-default set gs://my-bucket """ _EVENT_DESCRIPTION = """ <B>EVENT</B> The ``gsutil retention event`` command enables or disables an event-based hold on an object. <B>EXAMPLES</B> Setting the event-based hold on an object: gsutil retention event set gs://my-bucket/my-object Releasing the event-based hold on an object: gsutil retention event release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the event-based hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention event set gs://my-bucket/my-object If you want to set or release an event-based hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets an event-based hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention event set gs://my-bucket/.jpg """ _TEMP_DESCRIPTION = """ <B>TEMP</B> The ``gsutil retention temp`` command enables or disables a temporary hold on an object. <B>EXAMPLES</B> Setting the temporary hold on an object: gsutil retention temp set gs://my-bucket/my-object Releasing the temporary hold on an object: gsutil retention temp release gs://my-bucket/my-object You can also provide a precondition on an object's metageneration in order to avoid potential race conditions. You can use gsutil's '-h' option to specify preconditions. For example, the following specifies a precondition that checks an object's metageneration before setting the temporary hold on the object: gsutil -h "x-goog-if-metageneration-match: 1" \\ retention temp set gs://my-bucket/my-object If you want to set or release a temporary hold on a large number of objects, then you might want to use the top-level '-m' option to perform a parallel update. For example, the following command sets a temporary hold on objects ending with .jpg in parallel, in the root folder: gsutil -m retention temp set gs://bucket/.jpg """ _SYNOPSIS = (_SET_SYNOPSIS + _CLEAR_SYNOPSIS + _GET_SYNOPSIS + _LOCK_SYNOPSIS + _EVENT_DEFAULT_SYNOPSIS + _EVENT_SYNOPSIS + _TEMP_SYNOPSIS) _DESCRIPTION = (_SET_DESCRIPTION + _CLEAR_DESCRIPTION + _GET_DESCRIPTION + _LOCK_DESCRIPTION + _EVENT_DEFAULT_DESCRIPTION + _EVENT_DESCRIPTION + _TEMP_DESCRIPTION) _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _clear_help_text = CreateHelpText(_CLEAR_SYNOPSIS, _CLEAR_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _lock_help_text = CreateHelpText(_LOCK_SYNOPSIS, _LOCK_DESCRIPTION) _event_default_help_text = CreateHelpText(_EVENT_DEFAULT_SYNOPSIS, _EVENT_DEFAULT_DESCRIPTION) _event_help_text = CreateHelpText(_EVENT_SYNOPSIS, _EVENT_DESCRIPTION) _temp_help_text = CreateHelpText(_TEMP_SYNOPSIS, _TEMP_DESCRIPTION) class RetentionCommand(Command): """Implementation of gsutil retention command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'retention', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'clear': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], 'lock': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'event-default': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'event': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, 'temp': { 'set': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()], 'release': [CommandArgument.MakeZeroOrMoreCloudURLsArgument()] }, }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='retention', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Provides utilities to interact with Retention Policy feature.'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, 'clear': _clear_help_text, 'lock': _lock_help_text, 'event-default': _event_default_help_text, 'event': _event_help_text, 'temp': _temp_help_text }, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--retention-period={}s'.format( RetentionInSeconds(self.args[1])) ] + self.args[2:], # Adds target bucket URLs. flag_map={}), }, flag_map={}) # Don't trigger unneeded translation now that complete command is above. self.args = self.args[:1] else: gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'clear': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--clear-retention-period' ], flag_map={}, ), 'event': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'event-default': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--default-event-based-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-default-event-based-hold' ], flag_map={}, ), }, flag_map={}, ), 'get': GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'describe', '--format=yaml(retentionPolicy)', '--raw' ], flag_map={}), 'lock': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--lock-retention-period' ], flag_map={}, ), 'temp': GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--temporary-hold' ], flag_map={}, ), 'release': GcloudStorageMap( gcloud_command=[ 'storage', 'objects', 'update', '--no-temporary-hold' ], flag_map={}, ), }, flag_map={}, ), }, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the retention command.""" # If the only credential type the user supplies in their boto file is HMAC, # GetApiSelector logic will force us to use the XML API, which bucket lock # does not support at the moment. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException(('The {} command can only be used with the GCS ' 'JSON API. If you have only supplied hmac ' 'credentials in your boto file, please instead ' 'supply a credential type that can be used with ' 'the JSON API.').format(self.command_name)) self.preconditions = PreconditionsFromHeaders(self.headers) action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'set': func = self._SetRetention elif action_subcommand == 'clear': func = self._ClearRetention elif action_subcommand == 'get': func = self._GetRetention elif action_subcommand == 'lock': func = self._LockRetention elif action_subcommand == 'event-default': func = self._DefaultEventHold elif action_subcommand == 'event': func = self._EventHold elif action_subcommand == 'temp': func = self._TempHold else: raise CommandException( ('Invalid subcommand "{}" for the {} command.\n' 'See "gsutil help retention".').format(action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func() def BucketUpdateFunc(self, url_args, bucket_metadata_update, fields, log_msg_template): preconditions = Preconditions( meta_gen_match=self.preconditions.meta_gen_match) # Iterate over URLs, expanding wildcards and setting the new bucket metadata # on each bucket. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info(log_msg_template, blr) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata_update, preconditions=preconditions, provider=url.scheme, fields=fields) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def ObjectUpdateMetadataFunc(self, patch_obj_metadata, log_template, name_expansion_result, thread_state=None): """Updates metadata on an object using PatchObjectMetadata. Args: patch_obj_metadata: Metadata changes that should be applied to the existing object. log_template: The log template that should be printed for each object. name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info(log_template, exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetObjectNameExpansionIterator(self, url_args): return NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metageneration']) def _GetSeekAheadNameExpansionIterator(self, url_args): return SeekAheadNameExpansionIterator(self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) def _SetRetention(self): """Set retention retention_period on one or more buckets.""" seconds = RetentionInSeconds(self.args[0]) retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=seconds)) log_msg_template = 'Setting Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _ClearRetention(self): """Clear retention retention_period on one or more buckets.""" retention_policy = (apitools_messages.Bucket.RetentionPolicyValue( retentionPeriod=None)) log_msg_template = 'Clearing Retention Policy on %s...' bucket_metadata_update = apitools_messages.Bucket( retentionPolicy=retention_policy) url_args = self.args self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'retentionPolicy'], log_msg_template=log_msg_template) return 0 def _GetRetention(self): """Get Retention Policy for a single bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['retentionPolicy']) print(RetentionPolicyToString(bucket_metadata.retentionPolicy, bucket_url)) return 0 def _LockRetention(self): """Lock Retention Policy on one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and setting the Retention Policy # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True # Get bucket metadata to provide a precondition. bucket_metadata = self.gsutil_api.GetBucket( url.bucket_name, provider=url.scheme, fields=['id', 'metageneration', 'retentionPolicy']) if (not (bucket_metadata.retentionPolicy and bucket_metadata.retentionPolicy.retentionPeriod)): # TODO: implement '-c' flag to continue_on_error raise CommandException( 'Bucket "{}" does not have an Unlocked Retention Policy.'.format( url.bucket_name)) elif bucket_metadata.retentionPolicy.isLocked is True: self.logger.error('Retention Policy on "%s" is already locked.', blr) elif ConfirmLockRequest(url.bucket_name, bucket_metadata.retentionPolicy): self.logger.info('Locking Retention Policy on %s...', blr) self.gsutil_api.LockRetentionPolicy(url.bucket_name, bucket_metadata.metageneration, provider=url.scheme) else: self.logger.error( ' Abort Locking Retention Policy on {}'.format(blr)) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def _DefaultEventHold(self): """Sets default value for Event-Based Hold on one or more buckets.""" hold = None if self.args: if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention event-default"' ' command.\nSee "gsutil help retention event".').format( self.sub_opts)) verb = 'Setting' if hold else 'Releasing' log_msg_template = '{} default Event-Based Hold on %s...'.format(verb) bucket_metadata_update = apitools_messages.Bucket( defaultEventBasedHold=hold) url_args = self.args[1:] self.BucketUpdateFunc(url_args, bucket_metadata_update, fields=['id', 'defaultEventBasedHold'], log_msg_template=log_msg_template) return 0 def _EventHold(self): """Sets or unsets Event-Based Hold on one or more objects.""" sub_command_name = 'event' sub_command_full_name = 'Event-Based' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetEventHoldFuncWrapper if hold else ReleaseEventHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _TempHold(self): """Sets or unsets Temporary Hold on one or more objects.""" sub_command_name = 'temp' sub_command_full_name = 'Temporary' hold = self._ProcessHoldArgs(sub_command_name) url_args = self.args[1:] obj_metadata_update_wrapper = (SetTempHoldFuncWrapper if hold else ReleaseTempHoldFuncWrapper) self._SetHold(obj_metadata_update_wrapper, url_args, sub_command_full_name) return 0 def _ProcessHoldArgs(self, sub_command_name): """Processes command args for Temporary and Event-Based Hold sub-command. Args: sub_command_name: The name of the subcommand: "temp" / "event" Returns: Returns a boolean value indicating whether to set (True) or release (False)the Hold. """ hold = None if self.args[0].lower() == 'set': hold = True elif self.args[0].lower() == 'release': hold = False else: raise CommandException( ('Invalid subcommand "{}" for the "retention {}" command.\n' 'See "gsutil help retention {}".').format(self.args[0], sub_command_name, sub_command_name)) return hold def _SetHold(self, obj_metadata_update_wrapper, url_args, sub_command_full_name): """Common logic to set or unset Event-Based/Temporary Hold on objects. Args: obj_metadata_update_wrapper: The function for updating related fields in Object metadata. url_args: List of object URIs. sub_command_full_name: The full name for sub-command: "Temporary" / "Event-Based" """ if len(url_args) == 1 and not self.recursion_requested: url = StorageUrlFromString(url_args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL ({}) must name an object'.format( url_args[0])) name_expansion_iterator = self._GetObjectNameExpansionIterator(url_args) seek_ahead_iterator = self._GetSeekAheadNameExpansionIterator(url_args) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True try: # TODO: implement '-c' flag to continue_on_error # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(obj_metadata_update_wrapper, name_expansion_iterator, UpdateObjectMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException( '{} Hold for some objects could not be set.'.format( sub_command_full_name))	29.76 KB	2025-06-25 08:19
📄 rewrite.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of rewrite command (in-place cloud object transformation).""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import textwrap import time from apitools.base.py import encoding from boto import config from gslib.cloud_api import EncryptionException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.progress_callback import FileProgressCallbackHandler from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.encryption_helper import MAX_DECRYPTION_KEYS from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.text_util import ConvertRecursiveToFlatWildcard from gslib.utils.text_util import NormalizeStorageClass from gslib.utils import text_util from gslib.utils.translation_helper import PreconditionsFromHeaders MAX_PROGRESS_INDICATOR_COLUMNS = 65 _SYNOPSIS = """ gsutil rewrite -k [-O] [-f] [-r] [-s] url... gsutil rewrite -k [-O] [-f] [-r] [-s] -I gsutil rewrite -s [-k] [-O] [-f] [-r] url... gsutil rewrite -s [-k] [-O] [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rewrite command rewrites cloud objects, applying the specified transformations to them. The transformation(s) are atomic for each affected object and applied based on the input transformation flags. Object metadata values are preserved unless altered by a transformation. At least one transformation flag, -k or -s, must be included in the command. The -k flag is supported to add, rotate, or remove encryption keys on objects. For example, the command: gsutil rewrite -k -r gs://bucket updates all objects in gs://bucket with the current encryption key from your boto config file, which may either be a base64-encoded CSEK or the fully-qualified name of a Cloud KMS key. The rewrite command acts only on live object versions, so specifying a URL with a generation number fails. If you want to rewrite a noncurrent version, first copy it to the live version, then rewrite it, for example: gsutil cp gs://bucket/object#123 gs://bucket/object gsutil rewrite -k gs://bucket/object You can use the -s option to specify a new storage class for objects. For example, the command: gsutil rewrite -s nearline gs://bucket/foo rewrites the object, changing its storage class to nearline. If you specify the -k option and you have an encryption key set in your boto configuration file, the rewrite command skips objects that are already encrypted with the specified key. For example, if you run: gsutil rewrite -k -r gs://bucket and gs://bucket contains objects encrypted with the key specified in your boto configuration file, gsutil skips rewriting those objects and only rewrites objects that are not encrypted with the specified key. This avoids the cost of performing redundant rewrite operations. If you specify the -k option and you do not have an encryption key set in your boto configuration file, gsutil always rewrites each object, without explicitly specifying an encryption key. This results in rewritten objects being encrypted with either the bucket's default KMS key (if one is set) or Google-managed encryption (no CSEK or CMEK). Gsutil does not attempt to determine whether the operation is redundant (and thus skippable) because gsutil cannot be sure how the object is encrypted after the rewrite. Note that if your goal is to encrypt objects with a bucket's default KMS key, you can avoid redundant rewrite costs by specifying the bucket's default KMS key in your boto configuration file; this allows gsutil to perform an accurate comparison of the objects' current and desired encryption configurations and skip rewrites for objects already encrypted with that key. If have an encryption key set in your boto configuration file and specify multiple transformations, gsutil only skips those that would not change the object's state. For example, if you run: gsutil rewrite -s nearline -k -r gs://bucket and gs://bucket contains objects that already match the encryption configuration but have a storage class of standard, the only transformation applied to those objects would be the change in storage class. You can pass a list of URLs (one per line) to rewrite on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to rewrite objects identified by a program, such as: some_program \| gsutil -m rewrite -k -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. The rewrite command requires OWNER permissions on each object to preserve object ACLs. You can bypass this by using the -O flag, which causes gsutil not to read the object's ACL and instead apply the default object ACL to the rewritten object: gsutil rewrite -k -O -r gs://bucket <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when rewriting multiple objects. If some of the objects could not be rewritten, gsutil's exit status is non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rewrite ...". -I Causes gsutil to read the list of objects to rewrite from stdin. This allows you to run a program that generates the list of objects to rewrite. -k Rewrite objects with the current encryption key specified in your boto configuration file. The value for encryption_key may be either a base64-encoded CSEK or a fully-qualified KMS key name. If no value is specified for encryption_key, gsutil ignores this flag. Instead, rewritten objects are encrypted with the bucket's default KMS key, if one is set, or Google-managed encryption, if no default KMS key is set. -O When a bucket has uniform bucket-level access (UBLA) enabled, the -O flag is required and skips all ACL checks. When a bucket has UBLA disabled, the -O flag rewrites objects with the bucket's default object ACL instead of the existing object ACL. This is needed if you do not have OWNER permission on the object. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents to be rewritten recursively. -s <class> Rewrite objects using the specified storage class. """) def _RewriteExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) cls.op_failure_count += 1 def _RewriteFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RewriteFunc(name_expansion_result, thread_state=thread_state) def GenerationCheckGenerator(url_strs): """Generator function that ensures generation-less (live) arguments.""" for url_str in url_strs: if StorageUrlFromString(url_str).generation is not None: raise CommandException('"rewrite" called on URL with generation (%s).' % url_str) yield url_str class _TransformTypes(object): """Enum class for valid transforms.""" CRYPTO_KEY = 'crypto_key' STORAGE_CLASS = 'storage_class' class RewriteCommand(Command): """Implementation of gsutil rewrite command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rewrite', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='fkIrROs:', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rewrite', help_name_aliases=['rekey', 'rotate'], help_type='command_help', help_one_line_summary='Rewrite objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'objects', 'update'], flag_map={ '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), # Adding encryptions handled in shim_util.py. '-k': None if config.get('GSUtil', 'encryption_key', None) else GcloudStorageFlag('--clear-encryption-key'), '-O': GcloudStorageFlag('--no-preserve-acl'), '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-s': GcloudStorageFlag('-s'), }, ) def CheckProvider(self, url): if url.scheme != 'gs': raise CommandException( '"rewrite" called on URL with unsupported provider: %s' % str(url)) def RunCommand(self): """Command entry point for the rewrite command.""" self.continue_on_error = self.parallel_operations self.csek_hash_to_keywrapper = {} self.dest_storage_class = None self.no_preserve_acl = False self.read_args_from_stdin = False self.supported_transformation_flags = ['-k', '-s'] self.transform_types = set() self.op_failure_count = 0 self.boto_file_encryption_keywrapper = GetEncryptionKeyWrapper(config) self.boto_file_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256 if self.boto_file_encryption_keywrapper else None) if self.sub_opts: for o, a in self.sub_opts: if o == '-f': self.continue_on_error = True elif o == '-k': self.transform_types.add(_TransformTypes.CRYPTO_KEY) elif o == '-I': self.read_args_from_stdin = True elif o == '-O': self.no_preserve_acl = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True elif o == '-s': self.transform_types.add(_TransformTypes.STORAGE_CLASS) self.dest_storage_class = NormalizeStorageClass(a) if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rewrite command (without -I) expects at ' 'least one URL.') url_strs = self.args if not self.transform_types: raise CommandException( 'rewrite command requires at least one transformation flag. ' 'Currently supported transformation flags: %s' % self.supported_transformation_flags) self.preconditions = PreconditionsFromHeaders(self.headers or {}) url_strs_generator = GenerationCheckGenerator(url_strs) # Convert recursive flag to flat wildcard to avoid performing multiple # listings. if self.recursion_requested: url_strs_generator = ConvertRecursiveToFlatWildcard(url_strs_generator) # Expand the source argument(s). name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs_generator, self.recursion_requested, project_id=self.project_id, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=['name', 'size']) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: # Perform the same recursive-to-flat conversion on original url_strs so # that it is as true to the original iterator as possible. seek_ahead_url_strs = ConvertRecursiveToFlatWildcard(url_strs) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), seek_ahead_url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Rather than have each worker repeatedly calculate the sha256 hash for each # decryption_key in the boto config, do this once now and cache the results. for i in range(0, MAX_DECRYPTION_KEYS): key_number = i + 1 keywrapper = CryptoKeyWrapperFromKey( config.get('GSUtil', 'decryption_key%s' % str(key_number), None)) if keywrapper is None: # Stop at first attribute absence in lexicographical iteration. break if keywrapper.crypto_type == CryptoKeyType.CSEK: self.csek_hash_to_keywrapper[keywrapper.crypto_key_sha256] = keywrapper # Also include the encryption_key, since it should be used to decrypt and # then encrypt if the object's CSEK should remain the same. if self.boto_file_encryption_sha256 is not None: self.csek_hash_to_keywrapper[self.boto_file_encryption_sha256] = ( self.boto_file_encryption_keywrapper) if self.boto_file_encryption_keywrapper is None: msg = '\n'.join( textwrap.wrap( 'NOTE: No encryption_key was specified in the boto configuration ' 'file, so gsutil will not provide an encryption key in its rewrite ' 'API requests. This will decrypt the objects unless they are in ' 'buckets with a default KMS key set, in which case the service ' 'will automatically encrypt the rewritten objects with that key.') ) print('%s\n' % msg, file=sys.stderr) # Perform rewrite requests in parallel (-m) mode, if requested. self.Apply(_RewriteFuncWrapper, name_expansion_iterator, _RewriteExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count'], seek_ahead_iterator=seek_ahead_iterator) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be rewritten.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RewriteFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) transform_url = name_expansion_result.expanded_storage_url self.CheckProvider(transform_url) # Get all fields so that we can ensure that the target metadata is # specified correctly. src_metadata = gsutil_api.GetObjectMetadata( transform_url.bucket_name, transform_url.object_name, generation=transform_url.generation, provider=transform_url.scheme) if self.no_preserve_acl: # Leave ACL unchanged. src_metadata.acl = [] elif not src_metadata.acl: raise CommandException( 'No OWNER permission found for object %s. If your bucket has uniform ' 'bucket-level access (UBLA) enabled, include the -O option in your ' 'command to avoid this error. If your bucket does not use UBLA, you ' 'can use the -O option to apply the bucket\'s default object ACL ' 'when rewriting.' % transform_url) # Note: If other transform types are added, they must ensure that the # encryption key configuration matches the boto configuration, because # gsutil maintains an invariant that all objects it writes use the # encryption_key value (including decrypting if no key is present). # Store metadata about src encryption to make logic below easier to read. src_encryption_kms_key = (src_metadata.kmsKeyName if src_metadata.kmsKeyName else None) src_encryption_sha256 = None if (src_metadata.customerEncryption and src_metadata.customerEncryption.keySha256): src_encryption_sha256 = src_metadata.customerEncryption.keySha256 # In python3, hashes are bytes, use ascii since it should be ascii src_encryption_sha256 = src_encryption_sha256.encode('ascii') src_was_encrypted = (src_encryption_sha256 is not None or src_encryption_kms_key is not None) # Also store metadata about dest encryption. dest_encryption_kms_key = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CMEK): dest_encryption_kms_key = self.boto_file_encryption_keywrapper.crypto_key dest_encryption_sha256 = None if (self.boto_file_encryption_keywrapper is not None and self.boto_file_encryption_keywrapper.crypto_type == CryptoKeyType.CSEK): dest_encryption_sha256 = ( self.boto_file_encryption_keywrapper.crypto_key_sha256) should_encrypt_dest = self.boto_file_encryption_keywrapper is not None encryption_unchanged = (src_encryption_sha256 == dest_encryption_sha256 and src_encryption_kms_key == dest_encryption_kms_key) # Prevent accidental key rotation. if (_TransformTypes.CRYPTO_KEY not in self.transform_types and not encryption_unchanged): raise EncryptionException( 'The "-k" flag was not passed to the rewrite command, but the ' 'encryption_key value in your boto config file did not match the key ' 'used to encrypt the object "%s" (hash: %s). To encrypt the object ' 'using a different key, you must specify the "-k" flag.' % (transform_url, src_encryption_sha256)) # Determine if we can skip this rewrite operation (this should only be done # when ALL of the specified transformations are redundant). redundant_transforms = [] # STORAGE_CLASS transform is redundant if the target storage class matches # the existing storage class. if (_TransformTypes.STORAGE_CLASS in self.transform_types and self.dest_storage_class == NormalizeStorageClass( src_metadata.storageClass)): redundant_transforms.append('storage class') # CRYPTO_KEY transform is redundant if we're using the same encryption # key that was used to encrypt the source. However, if no encryption key was # specified, we should still perform the rewrite. This results in the # rewritten object either being encrypted with its bucket's default KMS key # or having no CSEK/CMEK encryption applied. While we could attempt fetching # the bucket's metadata and checking its default KMS key before performing # the rewrite (in the case where we appear to be transitioning from # no key to no key), that is vulnerable to the race condition where the # default KMS key is changed between when we check it and when we rewrite # the object. if (_TransformTypes.CRYPTO_KEY in self.transform_types and should_encrypt_dest and encryption_unchanged): redundant_transforms.append('encryption key') if len(redundant_transforms) == len(self.transform_types): self.logger.info('Skipping %s, all transformations were redundant: %s' % (transform_url, redundant_transforms)) return # First make a deep copy of the source metadata, then overwrite any # requested attributes (e.g. if a storage class change was specified). dest_metadata = encoding.PyValueToMessage( apitools_messages.Object, encoding.MessageToPyValue(src_metadata)) # Remove some unnecessary/invalid fields. dest_metadata.generation = None # Service has problems if we supply an ID, but it is responsible for # generating one, so it is not necessary to include it here. dest_metadata.id = None # Ensure we don't copy over the KMS key name or CSEK key info from the # source object; those should only come from the boto config's # encryption_key value. dest_metadata.customerEncryption = None dest_metadata.kmsKeyName = None # Both a storage class change and CMEK encryption should be set as part of # the dest object's metadata. CSEK encryption, if specified, is added to the # request later via headers obtained from the keywrapper value passed to # encryption_tuple. if _TransformTypes.STORAGE_CLASS in self.transform_types: dest_metadata.storageClass = self.dest_storage_class if dest_encryption_kms_key is not None: dest_metadata.kmsKeyName = dest_encryption_kms_key # Make sure we have the CSEK key necessary to decrypt. decryption_keywrapper = None if src_encryption_sha256 is not None: if src_encryption_sha256 in self.csek_hash_to_keywrapper: decryption_keywrapper = ( self.csek_hash_to_keywrapper[src_encryption_sha256]) else: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s' % (src_encryption_sha256, transform_url)) operation_name = 'Rewriting' if _TransformTypes.CRYPTO_KEY in self.transform_types: if src_was_encrypted and should_encrypt_dest: if not encryption_unchanged: operation_name = 'Rotating' # Else, keep "Rewriting". This might occur when -k was specified and was # redundant, but we're performing the operation anyway because some # other transformation was not redundant. elif src_was_encrypted and not should_encrypt_dest: operation_name = 'Decrypting' elif not src_was_encrypted and should_encrypt_dest: operation_name = 'Encrypting' # TODO: Remove this call (used to verify tests) and make it processed by # the UIThread. sys.stderr.write( _ConstructAnnounceText(operation_name, transform_url.url_string)) sys.stderr.flush() # Message indicating beginning of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=False, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=transform_url, operation_name=operation_name).call gsutil_api.CopyObject(src_metadata, dest_metadata, src_generation=transform_url.generation, preconditions=self.preconditions, progress_callback=progress_callback, decryption_tuple=decryption_keywrapper, encryption_tuple=self.boto_file_encryption_keywrapper, provider=transform_url.scheme, fields=[]) # Message indicating end of operation. gsutil_api.status_queue.put( FileMessage(transform_url, None, time.time(), finished=True, size=src_metadata.size, message_type=FileMessage.FILE_REWRITE)) def _ConstructAnnounceText(operation_name, url_string): """Constructs announce text for ongoing operations on url_string. This truncates the text to a maximum of MAX_PROGRESS_INDICATOR_COLUMNS, and informs the rewrite-related operation ('Encrypting', 'Rotating', or 'Decrypting'). Args: operation_name: String describing the operation, i.e. 'Rotating' or 'Encrypting'. url_string: String describing the file/object being processed. Returns: Formatted announce text for outputting operation progress. """ # Operation name occupies 10 characters (enough for 'Encrypting'), plus a # space. The rest is used for url_string. If a longer operation name is # used, it will be truncated. We can revisit this size if we need to support # a longer operation, but want to make sure the terminal output is meaningful. justified_op_string = operation_name[:10].ljust(11) start_len = len(justified_op_string) end_len = len(': ') if (start_len + len(url_string) + end_len > MAX_PROGRESS_INDICATOR_COLUMNS): ellipsis_len = len('...') url_string = '...%s' % url_string[-(MAX_PROGRESS_INDICATOR_COLUMNS - start_len - end_len - ellipsis_len):] base_announce_text = '%s%s:' % (justified_op_string, url_string) format_str = '{0:%ds}' % MAX_PROGRESS_INDICATOR_COLUMNS return format_str.format(base_announce_text)	25.7 KB	2025-06-25 08:19
📄 rm.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rm command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from gslib.cloud_api import BucketNotFoundException from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DecrementFailureCount from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_PREFIX from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import StdinIterator from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil rm [-f] [-r] url... gsutil rm [-f] [-r] -I """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: As part of verifying the existence of objects prior to deletion, ``gsutil rm`` makes ``GET`` requests to Cloud Storage for object metadata. These requests incur `network and operations charges <https://cloud.google.com/storage/pricing>`_. The gsutil rm command removes objects and/or buckets. For example, the following command removes the object ``kitten.png``: gsutil rm gs://bucket/kitten.png Use the -r option to specify recursive object deletion. For example, the following command removes gs://bucket/subdir and all objects and subdirectories under it: gsutil rm -r gs://bucket/subdir When working with versioning-enabled buckets, note that the -r option removes all object versions in the subdirectory. To remove only the live version of each object in the subdirectory, use the ` wildcard <https://cloud.google.com/storage/docs/wildcards>`_. The following command removes all versions of all objects in a bucket, and then deletes the bucket: gsutil rm -r gs://bucket To remove all objects and their versions from a bucket without deleting the bucket, use the ``-a`` option: gsutil rm -a gs://bucket/ If you have a large number of objects to remove, use the ``gsutil -m`` option, which enables multi-threading/multi-processing: gsutil -m rm -r gs://my_bucket/subdir You can pass a list of URLs (one per line) to remove on stdin instead of as command line arguments by using the -I option. This allows you to use gsutil in a pipeline to remove objects identified by a program, such as: some_program \| gsutil -m rm -I The contents of stdin can name cloud URLs and wildcards of cloud URLs. Note that ``gsutil rm`` refuses to remove files from the local file system. For example, this fails: gsutil rm .txt WARNING: Object removal cannot be undone. Cloud Storage is designed to give developers a high amount of flexibility and control over their data, and Google maintains strict controls over the processing and purging of deleted data. If you have concerns that your application software or your users may at some point erroneously delete or replace data, see `Options for controlling data lifecycles <https://cloud.google.com/storage/docs/control-data-lifecycles>`_ for ways to protect your data from accidental data deletion. <B>OPTIONS</B> -f Continues silently (without printing error messages) despite errors when removing multiple objects. If some of the objects could not be removed, gsutil's exit status will be non-zero even if this flag is set. Execution will still halt if an inaccessible bucket is encountered. This option is implicitly set when running "gsutil -m rm ...". -I Causes gsutil to read the list of objects to remove from stdin. This allows you to run a program that generates the list of objects to remove. -R, -r The -R and -r options are synonymous. Causes bucket or bucket subdirectory contents (all objects and subdirectories that it contains) to be removed recursively. If used with a bucket-only URL (like gs://bucket), after deleting objects and subdirectories gsutil deletes the bucket. This option implies the -a option and deletes all object versions. If you only want to delete live object versions, use the `* wildcard <https://cloud.google.com/storage/docs/wildcards>`_ instead of -r. -a Delete all versions of an object. """) def _RemoveExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" if not cls.continue_on_error: cls.logger.error(str(e)) # TODO: Use shared state to track missing bucket names when we get a # BucketNotFoundException. Then improve bucket removal logic and exception # messages. if isinstance(e, BucketNotFoundException): cls.bucket_not_found_count += 1 cls.logger.error(str(e)) else: if _ExceptionMatchesBucketToDelete(cls.bucket_strings_to_delete, e): DecrementFailureCount() else: cls.op_failure_count += 1 # pylint: disable=unused-argument def _RemoveFoldersExceptionHandler(cls, e): """When removing folders, we don't mind if none exist.""" if ((isinstance(e, CommandException) and NO_URLS_MATCHED_PREFIX in e.reason) or isinstance(e, NotFoundException)): DecrementFailureCount() else: raise e def _RemoveFuncWrapper(cls, name_expansion_result, thread_state=None): cls.RemoveFunc(name_expansion_result, thread_state=thread_state) def _ExceptionMatchesBucketToDelete(bucket_strings_to_delete, e): """Returns True if the exception matches a bucket slated for deletion. A recursive delete call on an empty bucket will raise an exception when listing its objects, but if we plan to delete the bucket that shouldn't result in a user-visible error. Args: bucket_strings_to_delete: Buckets slated for recursive deletion. e: Exception to check. Returns: True if the exception was a no-URLs-matched exception and it matched one of bucket_strings_to_delete, None otherwise. """ if bucket_strings_to_delete: msg = NO_URLS_MATCHED_TARGET % '' if msg in str(e): parts = str(e).split(msg) return len(parts) == 2 and parts[1] in bucket_strings_to_delete class RmCommand(Command): """Implementation of gsutil rm command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rm', command_name_aliases=['del', 'delete', 'remove'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=constants.NO_MAX, supported_sub_args='afIrR', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeZeroOrMoreCloudURLsArgument()]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rm', help_name_aliases=['del', 'delete', 'remove'], help_type='command_help', help_one_line_summary='Remove objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=['storage', 'rm'], flag_map={ '-r': GcloudStorageFlag('-r'), '-R': GcloudStorageFlag('-r'), '-a': GcloudStorageFlag('-a'), '-I': GcloudStorageFlag('-I'), '-f': GcloudStorageFlag('--continue-on-error'), }, ) def RunCommand(self): """Command entry point for the rm command.""" # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). self.continue_on_error = self.parallel_operations self.read_args_from_stdin = False self.all_versions = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-a': self.all_versions = True elif o == '-f': self.continue_on_error = True elif o == '-I': self.read_args_from_stdin = True elif o == '-r' or o == '-R': self.recursion_requested = True self.all_versions = True if self.read_args_from_stdin: if self.args: raise CommandException('No arguments allowed with the -I flag.') url_strs = StdinIterator() else: if not self.args: raise CommandException('The rm command (without -I) expects at ' 'least one URL.') url_strs = self.args # Tracks number of object deletes that failed. self.op_failure_count = 0 # Tracks if any buckets were missing. self.bucket_not_found_count = 0 # Tracks buckets that are slated for recursive deletion. bucket_urls_to_delete = [] self.bucket_strings_to_delete = [] if self.recursion_requested: bucket_fields = ['id'] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsBucket() or url.IsProvider(): for blr in self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields): bucket_urls_to_delete.append(blr.storage_url) self.bucket_strings_to_delete.append(url_str) self.preconditions = PreconditionsFromHeaders(self.headers or {}) try: # Expand wildcards, dirs, buckets, and bucket subdirs in URLs. name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, url_strs, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations) seek_ahead_iterator = None # Cannot seek ahead with stdin args, since we can only iterate them # once without buffering in memory. if not self.read_args_from_stdin: seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), url_strs, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) # Perform remove requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveExceptionHandler, fail_on_error=(not self.continue_on_error), shared_attrs=['op_failure_count', 'bucket_not_found_count'], seek_ahead_iterator=seek_ahead_iterator) # Assuming the bucket has versioning enabled, url's that don't map to # objects should throw an error even with all_versions, since the prior # round of deletes only sends objects to a history table. # This assumption that rm -a is only called for versioned buckets should be # corrected, but the fix is non-trivial. except CommandException as e: # Don't raise if there are buckets to delete -- it's valid to say: # gsutil rm -r gs://some_bucket # if the bucket is empty. if _ExceptionMatchesBucketToDelete(self.bucket_strings_to_delete, e): DecrementFailureCount() else: raise except ServiceException as e: if not self.continue_on_error: raise if self.bucket_not_found_count: raise CommandException('Encountered non-existent bucket during listing') if self.op_failure_count and not self.continue_on_error: raise CommandException('Some files could not be removed.') # If this was a gsutil rm -r command covering any bucket subdirs, # remove any dir_$folder$ objects (which are created by various web UI # tools to simulate folders). if self.recursion_requested: folder_object_wildcards = [] for url_str in url_strs: url = StorageUrlFromString(url_str) if url.IsObject(): folder_object_wildcards.append(url_str.rstrip('') + '_$folder$') if folder_object_wildcards: self.continue_on_error = True try: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, folder_object_wildcards, self.recursion_requested, project_id=self.project_id, all_versions=self.all_versions) # When we're removing folder objects, always continue on error self.Apply(_RemoveFuncWrapper, name_expansion_iterator, _RemoveFoldersExceptionHandler, fail_on_error=False) except CommandException as e: # Ignore exception from name expansion due to an absent folder file. if not e.reason.startswith(NO_URLS_MATCHED_PREFIX): raise # Now that all data has been deleted, delete any bucket URLs. for url in bucket_urls_to_delete: self.logger.info('Removing %s...', url) @Retry(NotEmptyException, tries=3, timeout_secs=1) def BucketDeleteWithRetry(): self.gsutil_api.DeleteBucket(url.bucket_name, provider=url.scheme) BucketDeleteWithRetry() if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be removed.' % (self.op_failure_count, plural_str, plural_str)) return 0 def RemoveFunc(self, name_expansion_result, thread_state=None): gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Removing %s...', exp_src_url) try: gsutil_api.DeleteObject(exp_src_url.bucket_name, exp_src_url.object_name, preconditions=self.preconditions, generation=exp_src_url.generation, provider=exp_src_url.scheme) except NotFoundException as e: # DeleteObject will sometimes return a 504 (DEADLINE_EXCEEDED) when # the operation was in fact successful. When a retry is attempted in # these cases, it will fail with a (harmless) 404. The 404 is harmless # since it really just means the file was already deleted, which is # what we want anyway. Here we simply downgrade the message to info # rather than error and correct the command-level failure total. self.logger.info('Cannot find %s', exp_src_url) DecrementFailureCount() _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time()))	16 KB	2025-06-25 08:19
📄 rpo.py # -- coding: utf-8 -- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the rpo command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap VALID_RPO_VALUES = ('ASYNC_TURBO', 'DEFAULT') VALID_RPO_VALUES_STRING = '({})'.format('\|'.join(VALID_RPO_VALUES)) _SET_SYNOPSIS = """ gsutil rpo set {} gs://<bucket_name>... """.format(VALID_RPO_VALUES_STRING) _GET_SYNOPSIS = """ gsutil rpo get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``rpo set`` command configures turbo replication for dual-region Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use turbo replication: gsutil rpo set ASYNC_TURBO gs://redbucket gs://bluebucket Configure your buckets to NOT use turbo replication: gsutil rpo set DEFAULT gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``rpo get`` command returns the replication setting for the specified Cloud Storage buckets. <B>GET EXAMPLES</B> Check if your buckets are using turbo replication: gsutil rpo get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``rpo`` command is used to retrieve or configure the `replication setting <https://cloud.google.com/storage/docs/availability-durability#cross-region-redundancy>`_ of dual-region Cloud Storage buckets. This command has two sub-commands: ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) GET_COMMAND = GcloudStorageMap(gcloud_command=[ 'storage', 'buckets', 'list', '--format=value[separator=": "](format("gs://{}", name),' 'rpo.yesno(no="None"))', '--raw' ], flag_map={}) SET_COMMAND = GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--recovery-point-objective' ], flag_map={}, ) class RpoCommand(Command): """Implements the gsutil rpo command.""" command_spec = Command.CreateCommandSpec( 'rpo', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1)], 'set': [ CommandArgument('mode', choices=list(VALID_RPO_VALUES)), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(), ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rpo', help_name_aliases=[], help_type='command_help', help_one_line_summary='Configure replication', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GET_COMMAND, 'set': SET_COMMAND, }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetRpo(self, blr): """Gets the rpo setting for a bucket.""" bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['rpo'], provider=bucket_url.scheme) rpo = bucket_metadata.rpo bucket = str(bucket_url).rstrip('/') print('%s: %s' % (bucket, rpo)) def _SetRpo(self, blr, rpo_value): """Sets the rpo setting for a bucket.""" bucket_url = blr.storage_url formatted_rpo_value = rpo_value if formatted_rpo_value not in VALID_RPO_VALUES: raise CommandException( 'Invalid value for rpo set.' ' Should be one of {}'.format(VALID_RPO_VALUES_STRING)) bucket_metadata = apitools_messages.Bucket(rpo=formatted_rpo_value) self.logger.info('Setting rpo %s for %s' % (formatted_rpo_value, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['rpo'], provider=bucket_url.scheme) return 0 def _Rpo(self): """Handles rpo command on Cloud Storage buckets.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('rpo only supports get\|set') subcommand_func = None subcommand_args = [] if subcommand == 'get': subcommand_func = self._GetRpo elif subcommand == 'set': subcommand_func = self._SetRpo setting_arg = self.args.pop(0) subcommand_args.append(setting_arg) # TODO: Remove this as rpo should work for XML as well. if self.gsutil_api.GetApiSelector('gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be with the Cloud Storage ' 'JSON API.') % self.command_name))) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: # TODO: Make this more general when adding RPO support for the XML API. if self.gsutil_api.GetApiSelector( bucket_listing_ref.storage_url.scheme) != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap(('The "%s" command can only be used for GCS ' 'buckets.') % self.command_name))) some_matched = True subcommand_func(bucket_listing_ref, *subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the rpo command.""" action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) return self._Rpo() else: raise CommandException('Invalid subcommand "%s", use get\|set instead.' % action_subcommand)	7.85 KB	2025-06-25 08:19
📄 rsync.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like rsync command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import errno import heapq import io from itertools import islice import logging import os import re import tempfile import textwrap import time import traceback import sys import six from six.moves import urllib from boto import config import crcmod from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import NotFoundException from gslib.cloud_api import ServiceException from gslib.command import Command from gslib.command import DummyArgChecker from gslib.commands.cp import ShimTranslatePredefinedAclSubOptForCopy from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import LogPerformanceSummaryParams from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import RegisterSignalHandler from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import constants from gslib.utils import copy_helper from gslib.utils import parallelism_framework_util from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.copy_helper import CreateCopyHelperOpts from gslib.utils.copy_helper import GetSourceFieldsNeededForCopy from gslib.utils.copy_helper import GZIP_ALL_FILES from gslib.utils.copy_helper import SkipUnsupportedObjectError from gslib.utils.hashing_helper import CalculateB64EncodedCrc32cFromContents from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import SLOW_CRCMOD_RSYNC_WARNING from gslib.utils.hashing_helper import SLOW_CRCMOD_WARNING from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import ConvertModeToBase8 from gslib.utils.posix_util import DeserializeFileAttributesFromObjectMetadata from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import InitializePreservePosixData from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import NA_ID from gslib.utils.posix_util import NA_MODE from gslib.utils.posix_util import NA_TIME from gslib.utils.posix_util import NeedsPOSIXAttributeUpdate from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import POSIXAttributes from gslib.utils.posix_util import SerializeFileAttributesToObjectMetadata from gslib.utils.posix_util import UID_ATTR from gslib.utils.posix_util import ValidateFilePermissionAccess from gslib.utils.posix_util import WarnFutureTimestamp from gslib.utils.posix_util import WarnInvalidValue from gslib.utils.posix_util import WarnNegativeAttribute from gslib.utils.rsync_util import DiffAction from gslib.utils.rsync_util import RsyncDiffToApply from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import CopyCustomMetadata from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.unit_util import TEN_MIB from gslib.wildcard_iterator import CreateWildcardIterator if six.PY3: long = int _SYNOPSIS = """ gsutil rsync [OPTION]... src_url dst_url """ # pylint: disable=anomalous-backslash-in-string _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil rsync command makes the contents under dst_url the same as the contents under src_url, by copying any missing files/objects (or those whose data has changed), and (if the -d option is specified) deleting any extra files/objects. src_url must specify a directory, bucket, or bucket subdirectory. For example, to sync the contents of the local directory "data" to the bucket gs://mybucket/data, you could do: gsutil rsync data gs://mybucket/data To recurse into directories use the -r option: gsutil rsync -r data gs://mybucket/data If you have a large number of objects to synchronize you might want to use the gsutil -m option (see "gsutil help options"), to perform parallel (multi-threaded/multi-processing) synchronization: gsutil -m rsync -r data gs://mybucket/data The -m option typically will provide a large performance boost if either the source or destination (or both) is a cloud URL. If both source and destination are file URLs the -m option will typically thrash the disk and slow synchronization down. Note 1: Shells (like bash, zsh) sometimes attempt to expand wildcards in ways that can be surprising. Also, attempting to copy files whose names contain wildcard characters can result in problems. For more details about these issues see `Wildcard behavior considerations <https://cloud.google.com/storage/docs/wildcards#surprising-behavior>`_. Note 2: If you are synchronizing a large amount of data between clouds you might consider setting up a `Google Compute Engine <https://cloud.google.com/products/compute-engine>`_ account and running gsutil there. Since cross-provider gsutil data transfers flow through the machine where gsutil is running, doing this can make your transfer run significantly faster than running gsutil on your local workstation. Note 3: rsync does not copy empty directory trees, since Cloud Storage uses a `flat namespace <https://cloud.google.com/storage/docs/folders>`_. <B>Using -d Option (with caution!) to mirror source and destination.</B> The rsync -d option is very useful and commonly used, because it provides a means of making the contents of a destination bucket or directory match those of a source bucket or directory. This is done by copying all data from the source to the destination and deleting all other data in the destination that is not in the source. Please exercise caution when you use this option: It's possible to delete large amounts of data accidentally if, for example, you erroneously reverse source and destination. To make the local directory my-data the same as the contents of gs://mybucket/data and delete objects in the local directory that are not in gs://mybucket/data: gsutil rsync -d -r gs://mybucket/data my-data To make the contents of gs://mybucket2 the same as gs://mybucket1 and delete objects in gs://mybucket2 that are not in gs://mybucket1: gsutil rsync -d -r gs://mybucket1 gs://mybucket2 You can also mirror data across local directories. This example will copy all objects from dir1 into dir2 and delete all objects in dir2 which are not in dir1: gsutil rsync -d -r dir1 dir2 To mirror your content across clouds: gsutil rsync -d -r gs://my-gs-bucket s3://my-s3-bucket Change detection works if the other Cloud provider is using md5 or CRC32. AWS multipart upload has an incompatible checksum. As mentioned above, using -d can be dangerous because of how quickly data can be deleted. For example, if you meant to synchronize a local directory from a bucket in the cloud but instead run the command: gsutil -m rsync -r -d ./your-dir gs://your-bucket and your-dir is currently empty, you will quickly delete all of the objects in gs://your-bucket. You can also cause large amounts of data to be lost quickly by specifying a subdirectory of the destination as the source of an rsync. For example, the command: gsutil -m rsync -r -d gs://your-bucket/data gs://your-bucket would cause most or all of the objects in gs://your-bucket to be deleted (some objects may survive if there are any with names that sort lower than "data" under gs://your-bucket/data). In addition to paying careful attention to the source and destination you specify with the rsync command, there are two more safety measures you can take when using gsutil rsync -d: 1. Try running the command with the rsync -n option first, to see what it would do without actually performing the operations. For example, if you run the command: gsutil -m rsync -r -d -n gs://your-bucket/data gs://your-bucket it will be immediately evident that running that command without the -n option would cause many objects to be deleted. 2. Enable object versioning in your bucket, which allows you to restore objects if you accidentally delete them. For more details see `Object Versioning <https://cloud.google.com/storage/docs/object-versioning>`_. <B>BE CAREFUL WHEN SYNCHRONIZING OVER OS-SPECIFIC FILE TYPES (SYMLINKS, DEVICES, ETC.)</B> Running gsutil rsync over a directory containing operating system-specific file types (symbolic links, device files, sockets, named pipes, etc.) can cause various problems. For example, running a command like: gsutil rsync -r ./dir gs://my-bucket will cause gsutil to follow any symbolic links in ./dir, creating objects in my-bucket containing the data from the files to which the symlinks point. This can cause various problems: * If you use gsutil rsync as a simple way to backup a directory to a bucket, restoring from that bucket will result in files where the symlinks used to be. At best this is wasteful of space, and at worst it can result in outdated data or broken applications -- depending on what is consuming the symlinks. * If you use gsutil rsync over directories containing broken symlinks, gsutil rsync will abort (unless you pass the -e option). * gsutil rsync skips symlinks that point to directories. Since gsutil rsync is intended to support data operations (like moving a data set to the cloud for computational processing) and it needs to be compatible both in the cloud and across common operating systems, there are no plans for gsutil rsync to support operating system-specific file types like symlinks. We recommend that users do one of the following: * Don't use gsutil rsync over directories containing symlinks or other OS- specific file types. * Use the -e option to exclude symlinks or the -x option to exclude OS-specific file types by name. * Use a tool (such as tar) that preserves symlinks and other OS-specific file types, packaging up directories containing such files before uploading to the cloud. <B>EVENTUAL CONSISTENCY WITH NON-GOOGLE CLOUD PROVIDERS</B> While Google Cloud Storage is strongly consistent, some cloud providers only support eventual consistency. You may encounter scenarios where rsync synchronizes using stale listing data when working with these other cloud providers. For example, if you run rsync immediately after uploading an object to an eventually consistent cloud provider, the added object may not yet appear in the provider's listing. Consequently, rsync will miss adding the object to the destination. If this happens you can rerun the rsync operation again later (after the object listing has "caught up"). <B>FAILURE HANDLING</B> The rsync command retries failures when it is useful to do so, but if enough failures happen during a particular copy or delete operation, or if a failure isn't retryable, the overall command fails. If the -C option is provided, the command instead skips failing objects and moves on. At the end of the synchronization run, if any failures were not successfully retried, the rsync command reports the count of failures and exits with non-zero status. At this point you can run the rsync command again, and gsutil attempts any remaining needed copy and/or delete operations. For more details about gsutil's retry handling, see `Retry strategy <https://cloud.google.com/storage/docs/retry-strategy#tools>`_. <B>CHANGE DETECTION ALGORITHM</B> To determine if a file or object has changed, gsutil rsync first checks whether the file modification time (mtime) of both the source and destination is available. If mtime is available at both source and destination, and the destination mtime is different than the source, or if the source and destination file size differ, gsutil rsync will update the destination. If the source is a cloud bucket and the destination is a local file system, and if mtime is not available for the source, gsutil rsync will use the time created for the cloud object as a substitute for mtime. Otherwise, if mtime is not available for either the source or the destination, gsutil rsync will fall back to using checksums. If the source and destination are both cloud buckets with checksums available, gsutil rsync will use these hashes instead of mtime. However, gsutil rsync will still update mtime at the destination if it is not present. If the source and destination have matching checksums and only the source has an mtime, gsutil rsync will copy the mtime to the destination. If neither mtime nor checksums are available, gsutil rsync will resort to comparing file sizes. Checksums will not be available when comparing composite Google Cloud Storage objects with objects at a cloud provider that does not support CRC32C (which is the only checksum available for composite objects). See 'gsutil help compose' for details about composite objects. <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B> If both the source and destination URL are cloud URLs from the same provider, gsutil copies data "in the cloud" (i.e., without downloading to and uploading from the machine where you run gsutil). In addition to the performance and cost advantages of doing this, copying in the cloud preserves metadata (like Content-Type and Cache-Control). In contrast, when you download data from the cloud it ends up in a file, which has no associated metadata, other than file modification time (mtime). Thus, unless you have some way to hold on to or re-create that metadata, synchronizing a bucket to a directory in the local file system will not retain the metadata other than mtime. Note that by default, the gsutil rsync command does not copy the ACLs of objects being synchronized and instead will use the default bucket ACL (see "gsutil help defacl"). You can override this behavior with the -p option. See the `Options section <https://cloud.google.com/storage/docs/gsutil/commands/rsync#options>`_ to learn how. <B>LIMITATIONS</B> 1. The gsutil rsync command will only allow non-negative file modification times to be used in its comparisons. This means gsutil rsync will resort to using checksums for any file with a timestamp before 1970-01-01 UTC. 2. The gsutil rsync command considers only the live object version in the source and destination buckets when deciding what to copy / delete. If versioning is enabled in the destination bucket then gsutil rsync's replacing or deleting objects will end up creating versions, but the command doesn't try to make any noncurrent versions match in the source and destination buckets. 3. The gsutil rsync command does not support copying special file types such as sockets, device files, named pipes, or any other non-standard files intended to represent an operating system resource. If you run gsutil rsync on a source directory that includes such files (for example, copying the root directory on Linux that includes /dev ), you should use the -x flag to exclude these files. Otherwise, gsutil rsync may fail or hang. 4. The gsutil rsync command copies changed files in their entirety and does not employ the `rsync delta-transfer algorithm <https://rsync.samba.org/tech_report/>`_ to transfer portions of a changed file. This is because Cloud Storage objects are immutable and no facility exists to read partial object checksums or perform partial replacements. <B>OPTIONS</B> -a predef-acl Sets the specified predefined ACL on uploaded objects. See "gsutil help acls" for further details. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Also see the -p option below. -c Causes the rsync command to compute and compare checksums (instead of comparing mtime) for files if the size of source and destination match. This option increases local disk I/O and run time if either src_url or dst_url are on the local file system. -C If an error occurs, continue to attempt to copy the remaining files. If errors occurred, gsutil's exit status will be non-zero even if this flag is set. This option is implicitly set when running "gsutil -m rsync...". NOTE: -C only applies to the actual copying operation. If an error occurs while iterating over the files in the local directory (e.g., invalid Unicode file name) gsutil will print an error message and abort. -d Delete extra files under dst_url not found under src_url. By default extra files are not deleted. NOTE: this option can delete data quickly if you specify the wrong source/destination combination. See the help section above, "BE CAREFUL WHEN USING -d OPTION!". -e Exclude symlinks. When specified, symbolic links will be ignored. Note that gsutil does not follow directory symlinks, regardless of whether -e is specified. -i Skip copying any files that already exist at the destination, regardless of their modification time. -j <ext,...> Applies gzip transport encoding to any file upload whose extension matches the -j extension list. This is useful when uploading files with compressible content (such as .js, .css, or .html files) because it saves network bandwidth while also leaving the data uncompressed in Google Cloud Storage. When you specify the -j option, files being uploaded are compressed in-memory and on-the-wire only. Both the local files and Cloud Storage objects remain uncompressed. The uploaded objects retain the Content-Type and name of the original files. Note that if you want to use the top-level -m option to parallelize copies along with the -j/-J options, your performance may be bottlenecked by the "max_upload_compression_buffer_size" boto config option, which is set to 2 GiB by default. This compression buffer size can be changed to a higher limit, e.g.: gsutil -o "GSUtil:max_upload_compression_buffer_size=8G" \\ -m rsync -j html,txt /local/source/dir gs://bucket/path -J Applies gzip transport encoding to file uploads. This option works like the -j option described above, but it applies to all uploaded files, regardless of extension. CAUTION: If you use this option and some of the source files don't compress well (e.g., that's often true of binary data), this option may result in longer uploads. -n Causes rsync to run in "dry run" mode, i.e., just outputting what would be copied or deleted without actually doing any copying/deleting. -p Causes ACLs to be preserved when objects are copied. Note that rsync will decide whether or not to perform a copy based only on object size and modification time, not current ACL state. Thus, if the source and destination differ in size or modification time and you run gsutil rsync -p, the file will be copied and ACL preserved. However, if the source and destination don't differ in size or checksum but have different ACLs, running gsutil rsync -p will have no effect. Note that this option has performance and cost implications when using the XML API, as it requires separate HTTP calls for interacting with ACLs. The performance issue can be mitigated to some degree by using gsutil -m rsync to cause parallel synchronization. Also, this option only works if you have OWNER access to all of the objects that are copied. You can avoid the additional performance and cost of using rsync -p if you want all objects in the destination bucket to end up with the same ACL by setting a default object ACL on that bucket instead of using rsync -p. See 'gsutil help defacl'. -P Causes POSIX attributes to be preserved when objects are copied. With this feature enabled, gsutil rsync will copy fields provided by stat. These are the user ID of the owner, the group ID of the owning group, the mode (permissions) of the file, and the access/modification timestamps of the file. For downloads, these attributes will only be set if the source objects were uploaded with this flag enabled. On Windows, this flag will only set and restore access time and modification time. This is because Windows doesn't have a notion of POSIX uid/gid/mode. -R, -r The -R and -r options are synonymous. Causes directories, buckets, and bucket subdirectories to be synchronized recursively. If you neglect to use this option gsutil will make only the top-level directory in the source and destination URLs match, skipping any sub-directories. -u When a file/object is present in both the source and destination, if mtime is available for both, do not perform the copy if the destination mtime is newer. -U Skip objects with unsupported object types instead of failing. Unsupported object types are Amazon S3 Objects in the GLACIER storage class. -x pattern Causes files/objects matching pattern to be excluded, i.e., any matching files/objects are not copied or deleted. Note that the pattern is a `Python regular expression <https://docs.python.org/3/howto/regex.html>`_, not a wildcard (so, matching any string ending in "abc" would be specified using ".abc$" rather than "abc"). Note also that the exclude path is always relative (similar to Unix rsync or tar exclude options). For example, if you run the command: gsutil rsync -x "data.[/\\].\\.txt$" dir gs://my-bucket it skips the file dir/data1/a.txt. You can use regex alternation to specify multiple exclusions, for example: gsutil rsync -x ".\\.txt$\|.\\.jpg$" dir gs://my-bucket skips all .txt and .jpg files in dir. NOTE: When using the Windows cmd.exe command line interpreter, use ``^`` as an escape character instead of ``\\`` and escape the ``\|`` character. When using Windows PowerShell, use ``'`` instead of ``"`` and surround the ``\|`` character with ``"``. -y pattern Similar to the -x option, but the command will first skip directories/prefixes using the provided pattern and then exclude files/objects using the same pattern. This is usually much faster, but won't work as intended with negative lookahead patterns. For example, if you run the command: gsutil rsync -y "^(?!.\\.txt$)." dir gs://my-bucket This would first exclude all subdirectories unless they end in .txt before excluding all files except those ending in .txt. Running the same command with the -x option would result in all .txt files being included, regardless of whether they appear in subdirectories that end in .txt. """) # pylint: enable=anomalous-backslash-in-string _NA = '-' _OUTPUT_BUFFER_SIZE = 64 1024 _PROGRESS_REPORT_LISTING_COUNT = 10000 # Tracks files we need to clean up at end or if interrupted. Because some # files are passed to rsync's diff iterators, it is difficult to manage when # they should be closed, especially in the event that we receive a signal to # exit. Every time such a file is opened, its file object should be appended # to this list. _tmp_files = [] # pylint: disable=unused-argument def _HandleSignals(signal_num, cur_stack_frame): """Called when rsync command is killed with SIGINT, SIGQUIT or SIGTERM.""" CleanUpTempFiles() def CleanUpTempFiles(): """Cleans up temp files. This function allows the main (RunCommand) function to clean up at end of operation, or if gsutil rsync is interrupted (e.g., via ^C). This is necessary because tempfile.NamedTemporaryFile doesn't allow the created file to be re-opened in read mode on Windows, so we have to use tempfile.mkstemp, which doesn't automatically delete temp files. """ # First pass: Close all the files. Wrapped iterators result in open file # objects for the same file, and Windows does not allow removing the file # at a given path until all its open file handles have been closed. for fileobj in _tmp_files: # Windows requires temp files to be closed before unlinking. if not fileobj.closed: fileobj.close() # Second pass: Remove each file, skipping duplicates that have already been # removed. for fileobj in _tmp_files: if os.path.isfile(fileobj.name): try: os.unlink(fileobj.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to close and delete temp file "%s". Got an error:\n%s', fileobj.name, e) def _DiffToApplyArgChecker(command_instance, diff_to_apply): """Arg checker that skips symlinks if -e flag specified.""" if (diff_to_apply.diff_action == DiffAction.REMOVE or not command_instance.exclude_symlinks): # No src URL is populated for REMOVE actions. return True exp_src_url = StorageUrlFromString(diff_to_apply.src_url_str) if exp_src_url.IsFileUrl() and os.path.islink(exp_src_url.object_name): command_instance.logger.info('Skipping symbolic link %s...', exp_src_url) return False return True def _ComputeNeededFileChecksums(logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5): """Computes any file checksums needed by _CompareObjects. Args: logger: logging.logger for outputting log messages. src_url_str: Source URL string. src_size: Source size src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: (src_crc32c, src_md5, dst_crc32c, dst_md5) """ src_url = StorageUrlFromString(src_url_str) dst_url = StorageUrlFromString(dst_url_str) if src_url.IsFileUrl(): if dst_crc32c != _NA or dst_url.IsFileUrl(): if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif dst_md5 != _NA or dst_url.IsFileUrl(): if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', src_url_str) with open(src_url.object_name, 'rb') as fp: src_md5 = CalculateB64EncodedMd5FromContents(fp) if dst_url.IsFileUrl(): if src_crc32c != _NA: if src_size > TEN_MIB: logger.info('Computing CRC32C for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_crc32c = CalculateB64EncodedCrc32cFromContents(fp) elif src_md5 != _NA: if dst_size > TEN_MIB: logger.info('Computing MD5 for %s...', dst_url_str) with open(dst_url.object_name, 'rb') as fp: dst_md5 = CalculateB64EncodedMd5FromContents(fp) return (src_crc32c, src_md5, dst_crc32c, dst_md5) def _ListUrlRootFunc(cls, args_tuple, thread_state=None): """Worker function for listing files/objects under to be sync'd. Outputs sorted list to out_file_name, formatted per _BuildTmpOutputLine. We sort the listed URLs because we don't want to depend on consistent sort order across file systems and cloud providers. Args: cls: Command instance. args_tuple: (base_url_str, out_file_name, desc), where base_url_str is top-level URL string to list; out_filename is name of file to which sorted output should be written; desc is 'source' or 'destination'. thread_state: gsutil Cloud API instance to use. """ gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) (base_url_str, out_filename, desc) = args_tuple # We sort while iterating over base_url_str, allowing parallelism of batched # sorting with collecting the listing. out_file = io.open(out_filename, mode='w', encoding=constants.UTF8) try: _BatchSort(_FieldedListingIterator(cls, gsutil_api, base_url_str, desc), out_file) except Exception as e: # pylint: disable=broad-except # Abandon rsync if an exception percolates up to this layer - retryable # exceptions are handled in the lower layers, so we got a non-retryable # exception (like 404 bucket not found) and proceeding would either be # futile or could result in data loss - for example: # gsutil rsync -d gs://non-existent-bucket ./localdir # would delete files from localdir. cls.logger.error('Caught non-retryable exception while listing %s: %s' % (base_url_str, e)) # Also print the full stack trace in debugging mode. This makes debugging # a bit easier. cls.logger.debug(traceback.format_exc()) cls.non_retryable_listing_failures = 1 out_file.close() def _LocalDirIterator(base_url): """A generator that yields a BLR for each file in a local directory. We use this function instead of WildcardIterator for listing a local directory without recursion, because the glob.globi implementation called by WildcardIterator skips "dot" files (which we don't want to do when synchronizing to or from a local directory). Args: base_url: URL for the directory over which to iterate. Yields: BucketListingObject for each file in the directory. """ for filename in os.listdir(base_url.object_name): filename = os.path.join(base_url.object_name, filename) if os.path.isfile(filename): yield BucketListingObject(StorageUrlFromString(filename), None) def _FieldedListingIterator(cls, gsutil_api, base_url_str, desc): """Iterator over base_url_str formatting output per _BuildTmpOutputLine. Args: cls: Command instance. gsutil_api: gsutil Cloud API instance to use for bucket listing. base_url_str: The top-level URL string over which to iterate. desc: 'source' or 'destination'. Yields: Output line formatted per _BuildTmpOutputLine. """ base_url = StorageUrlFromString(base_url_str) if base_url.scheme == 'file' and not cls.recursion_requested: iterator = _LocalDirIterator(base_url) else: if cls.recursion_requested: wildcard = '%s/*' % base_url_str.rstrip('/\\') else: wildcard = '%s/' % base_url_str.rstrip('/\\') fields = [ 'crc32c', 'md5Hash', 'name', 'size', 'timeCreated', 'metadata/%s' % MTIME_ATTR, ] if cls.preserve_posix_attrs: fields.extend([ 'metadata/%s' % ATIME_ATTR, 'metadata/%s' % MODE_ATTR, 'metadata/%s' % GID_ATTR, 'metadata/%s' % UID_ATTR, ]) exclude_tuple = ( base_url, cls.exclude_dirs, cls.exclude_pattern) if cls.exclude_pattern is not None else None iterator = CreateWildcardIterator( wildcard, gsutil_api, project_id=cls.project_id, exclude_tuple=exclude_tuple, ignore_symlinks=cls.exclude_symlinks, logger=cls.logger).IterObjects( # Request just the needed fields, to reduce bandwidth usage. bucket_listing_fields=fields) i = 0 for blr in iterator: # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. # We need to filter them out here, otherwise if the user tries to rsync # from GCS to a local directory it will result in a directory/file # conflict (e.g., trying to download an object called "mydata/" where the # local directory "mydata" exists). url = blr.storage_url if IsCloudSubdirPlaceholder(url, blr=blr): # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. continue if (cls.exclude_symlinks and url.IsFileUrl() and os.path.islink(url.object_name)): continue if cls.exclude_pattern: # The wildcard_iterator may optionally use the exclude pattern to exclude # directories while this section excludes individual files. str_to_check = url.url_string[len(base_url.url_string):] if str_to_check.startswith(url.delim): str_to_check = str_to_check[1:] if cls.exclude_pattern.match(str_to_check): continue i += 1 if i % _PROGRESS_REPORT_LISTING_COUNT == 0: cls.logger.info('At %s listing %d...', desc, i) yield _BuildTmpOutputLine(blr) def _BuildTmpOutputLine(blr): """Builds line to output to temp file for given BucketListingRef. Args: blr: The BucketListingRef. Returns: The output line, formatted as _EncodeUrl(URL)<sp>size<sp>time_created<sp>atime<sp>mtime<sp>mode<sp>uid<sp> gid<sp>crc32c<sp>md5 where md5 will only be present for cloud URLs that aren't composite objects. A missing field is populated with '-', or -1 in the case of atime/mtime/time_created. """ atime = NA_TIME crc32c = _NA gid = NA_ID md5 = _NA mode = NA_MODE mtime = NA_TIME time_created = NA_TIME uid = NA_ID url = blr.storage_url if url.IsFileUrl(): mode, _, _, _, uid, gid, size, atime, mtime, _ = os.stat(url.object_name) # atime/mtime can be a float, so it needs to be converted to a long. atime = long(atime) mtime = long(mtime) mode = ConvertModeToBase8(mode) # Don't use atime / mtime with times older than 1970-01-01 UTC. if atime < 0: atime = NA_TIME if mtime < 0: mtime = NA_TIME elif url.IsCloudUrl(): size = blr.root_object.size if blr.root_object.metadata is not None: found_m, mtime_str = GetValueFromObjectCustomMetadata( blr.root_object, MTIME_ATTR, NA_TIME) try: # The mtime value can be changed in the online console, this performs a # sanity check and sets the mtime to NA_TIME if it fails. mtime = long(mtime_str) if found_m and mtime <= NA_TIME: WarnNegativeAttribute('mtime', url.url_string) if mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', url.url_string) except ValueError: # Since mtime is a string, catch the case where it can't be cast as a # long. WarnInvalidValue('mtime', url.url_string) mtime = NA_TIME posix_attrs = DeserializeFileAttributesFromObjectMetadata( blr.root_object, url.url_string) mode = posix_attrs.mode.permissions atime = posix_attrs.atime uid = posix_attrs.uid gid = posix_attrs.gid # Sanitize the timestamp returned, and put it in UTC format. For more # information see the UTC class in gslib/util.py. time_created = ConvertDatetimeToPOSIX(blr.root_object.timeCreated) crc32c = blr.root_object.crc32c or _NA md5 = blr.root_object.md5Hash or _NA else: raise CommandException('Got unexpected URL type (%s)' % url.scheme) attrs = [ _EncodeUrl(url.url_string), # binary str in py2 / unicode str py 3 size, # int time_created, # int atime, # long mtime, # long mode, # int uid, # int gid, # int crc32c, # unicode md5, # unicode ] attrs = [six.ensure_text(str(i)) for i in attrs] return ' '.join(attrs) + '\n' def _EncodeUrl(url_string): """Encodes url_str with quote plus encoding and UTF8 character encoding. We use this for all URL encodings. Args: url_string (unicode): String URL to encode. Returns: (str) A string encoded using urllib's `quote_plus()` method. """ # N.B.: `quote_plus()` raises an error for unicode characters like è if you # don't pass it the language-appropriate string type. If you pass it `unicode` # in Python 2 or `bytes` in Python 3, it leads to surprising behavior for text # containing unicode chars. url_string = six.ensure_str(url_string) return urllib.parse.quote_plus(url_string, safe=b'~') def _DecodeUrl(enc_url_string): """Inverts encoding from `_EncodeUrl()`. Args: enc_url_string (str): String containing UTF-8-decodable characters that were encoded using urllib's `quote_plus()`. Returns: (unicode) A decoded URL. """ url = urllib.parse.unquote_plus(enc_url_string) if six.PY2: url = url.decode(constants.UTF8) return url # pylint: disable=bare-except def _BatchSort(in_iter, out_file): """Sorts input lines from in_iter and outputs to out_file. Sorts in batches as input arrives, so input file does not need to be loaded into memory all at once. Derived from Python Recipe 466302: Sorting big files the Python 2.4 way by Nicolas Lehuen. Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line when we could just sort on the first record (URL); but the sort order is identical either way. Args: in_iter: Input iterator. out_file: Output file. """ # Note: If chunk_files gets very large we can run out of open FDs. See .boto # file comments about rsync_buffer_lines. If increasing rsync_buffer_lines # doesn't suffice (e.g., for someone synchronizing with a really large # bucket), an option would be to make gsutil merge in passes, never # opening all chunk files simultaneously. buffer_size = config.getint('GSUtil', 'rsync_buffer_lines', 32000) chunk_files = [] try: while True: current_chunk = sorted(islice(in_iter, buffer_size)) if not current_chunk: break output_chunk = io.open('%s-%06i' % (out_file.name, len(chunk_files)), mode='w+', encoding=constants.UTF8) chunk_files.append(output_chunk) output_chunk.write(six.text_type(''.join(current_chunk))) output_chunk.flush() output_chunk.seek(0) out_file.writelines(heapq.merge(chunk_files)) except IOError as e: if e.errno == errno.EMFILE: raise CommandException('\n'.join( textwrap.wrap( 'Synchronization failed because too many open file handles were ' 'needed while building synchronization state. Please see the ' 'comments about rsync_buffer_lines in your .boto config file for a ' 'possible way to address this problem.'))) raise finally: for chunk_file in chunk_files: try: chunk_file.close() os.remove(chunk_file.name) except Exception as e: # pylint: disable=broad-except logging.debug( 'Failed to remove rsync chunk file "%s". Got an error:\n%s', chunk_file.name, e) class _DiffIterator(object): """Iterator yielding sequence of RsyncDiffToApply objects.""" def __init__(self, command_obj, base_src_url, base_dst_url): self.command_obj = command_obj self.compute_file_checksums = command_obj.compute_file_checksums self.delete_extras = command_obj.delete_extras self.recursion_requested = command_obj.recursion_requested self.logger = self.command_obj.logger self.base_src_url = base_src_url self.base_dst_url = base_dst_url self.preserve_posix = command_obj.preserve_posix_attrs self.skip_old_files = command_obj.skip_old_files self.ignore_existing = command_obj.ignore_existing self.logger.info('Building synchronization state...') # Files to track src and dst state should be created in the system's # preferred temp directory so that they are eventually cleaned up if our # cleanup callback is interrupted. temp_src_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-src-', delete=False) temp_dst_file = tempfile.NamedTemporaryFile(prefix='gsutil-rsync-dst-', delete=False) self.sorted_list_src_file_name = temp_src_file.name self.sorted_list_dst_file_name = temp_dst_file.name _tmp_files.append(temp_src_file) _tmp_files.append(temp_dst_file) # Close the files, but don't delete them. Because Windows does not allow # a temporary file to be reopened until it's been closed, we close the # files before proceeding. This allows each step below to open the file at # the specified path, perform I/O, and close it so that the next step may # do the same thing. temp_src_file.close() temp_dst_file.close() # Build sorted lists of src and dst URLs in parallel. To do this, pass # args to _ListUrlRootFunc as tuple (base_url_str, out_filename, desc) # where base_url_str is the starting URL string for listing. args_iter = iter([ ( self.base_src_url.url_string, self.sorted_list_src_file_name, 'source', ), ( self.base_dst_url.url_string, self.sorted_list_dst_file_name, 'destination', ), ]) # Contains error message from non-retryable listing failure. command_obj.non_retryable_listing_failures = 0 shared_attrs = ['non_retryable_listing_failures'] command_obj.Apply( _ListUrlRootFunc, args_iter, _RootListingExceptionHandler, shared_attrs, arg_checker=DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SPEED, fail_on_error=True) if command_obj.non_retryable_listing_failures: raise CommandException('Caught non-retryable exception - aborting rsync') # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open(self.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open(self.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) if (base_src_url.IsCloudUrl() and base_dst_url.IsFileUrl() and self.preserve_posix): self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self._ValidateObjectAccess() # Reset our file pointers to the beginning. self.sorted_list_src_file.seek(0) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) def _ValidateObjectAccess(self): """Validates that the user won't lose access to the files if copied. Iterates over the src file list to check if access will be maintained. If at any point we would orphan a file, a list of errors is compiled and logged with an exception raised to the user. """ errors = collections.deque() for src_url in self.sorted_src_urls_it: src_url_str, _, _, _, _, src_mode, src_uid, src_gid, _, _ = ( self._ParseTmpFileLine(src_url)) valid, err = ValidateFilePermissionAccess(src_url_str, uid=src_uid, gid=src_gid, mode=src_mode) if not valid: errors.append(err) if errors: for err in errors: self.logger.critical(err) raise CommandException('This sync will orphan file(s), please fix their ' 'permissions before trying again.') def _ParseTmpFileLine(self, line): """Parses output from _BuildTmpOutputLine. Parses into tuple: (URL, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) where crc32c and/or md5 can be _NA and atime/mtime/time_created can be NA_TIME. Args: line: The line to parse. Returns: Parsed tuple: (url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) """ (encoded_url, size, time_created, atime, mtime, mode, uid, gid, crc32c, md5) = line.rsplit(None, 9) return ( _DecodeUrl(encoded_url), int(size), long(time_created), long(atime), long(mtime), int(mode), int(uid), int(gid), crc32c, md5.strip(), ) def _WarnIfMissingCloudHash(self, url_str, crc32c, md5): """Warns if given url_str is a cloud URL and is missing both crc32c and md5. Args: url_str: Destination URL string. crc32c: Destination CRC32c. md5: Destination MD5. Returns: True if issued warning. """ # One known way this can currently happen is when rsync'ing objects larger # than 5 GB from S3 (for which the etag is not an MD5). if (StorageUrlFromString(url_str).IsCloudUrl() and crc32c == _NA and md5 == _NA): self.logger.warn( 'Found no hashes to validate %s. Integrity cannot be assured without ' 'hashes.', url_str) return True return False def _CompareObjects( self, src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5, ): """Returns whether src should replace dst object, and if mtime is present. Uses mtime, size, or whatever checksums are available. Args: src_url_str: Source URL string. src_size: Source size. src_mtime: Source modification time. src_crc32c: Source CRC32c. src_md5: Source MD5. dst_url_str: Destination URL string. dst_size: Destination size. dst_mtime: Destination modification time. dst_crc32c: Destination CRC32c. dst_md5: Destination MD5. Returns: A 3-tuple indicating if src should replace dst, and if src and dst have mtime. """ # Note: This function is called from __iter__, which is called from the # Command.Apply driver. Thus, all checksum computation will be run in a # single thread, which is good (having multiple threads concurrently # computing checksums would thrash the disk). # # Comparison Hierarchy: # 1. mtime # 2. md5/crc32c hashes (if available) # 3. size has_src_mtime = src_mtime > NA_TIME has_dst_mtime = dst_mtime > NA_TIME use_hashes = (self.compute_file_checksums or (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsCloudUrl())) if self.ignore_existing: return False, has_src_mtime, has_dst_mtime if (self.skip_old_files and has_src_mtime and has_dst_mtime and src_mtime < dst_mtime): return False, has_src_mtime, has_dst_mtime if not use_hashes and has_src_mtime and has_dst_mtime: return (src_mtime != dst_mtime or src_size != dst_size, has_src_mtime, has_dst_mtime) if src_size != dst_size: return True, has_src_mtime, has_dst_mtime src_crc32c, src_md5, dst_crc32c, dst_md5 = _ComputeNeededFileChecksums( self.logger, src_url_str, src_size, src_crc32c, src_md5, dst_url_str, dst_size, dst_crc32c, dst_md5, ) if src_md5 != _NA and dst_md5 != _NA: self.logger.debug('Comparing md5 for %s and %s', src_url_str, dst_url_str) return src_md5 != dst_md5, has_src_mtime, has_dst_mtime if src_crc32c != _NA and dst_crc32c != _NA: self.logger.debug('Comparing crc32c for %s and %s', src_url_str, dst_url_str) return src_crc32c != dst_crc32c, has_src_mtime, has_dst_mtime if not self._WarnIfMissingCloudHash(src_url_str, src_crc32c, src_md5): self._WarnIfMissingCloudHash(dst_url_str, dst_crc32c, dst_md5) # Without checksums or mtime to compare we depend only on basic size # comparison. return False, has_src_mtime, has_dst_mtime def __iter__(self): """Iterates over src/dst URLs and produces a RsyncDiffToApply sequence. Yields: The RsyncDiffToApply. """ # Strip trailing slashes, if any, so we compute tail length against # consistent position regardless of whether trailing slashes were included # or not in URL. base_src_url_len = len(self.base_src_url.url_string.rstrip('/\\')) base_dst_url_len = len(self.base_dst_url.url_string.rstrip('/\\')) out_of_src_items = False src_url_str = dst_url_str = None # Invariant: After each yield, the URLs in src_url_str, dst_url_str, # self.sorted_src_urls_it, and self.sorted_dst_urls_it are not yet # processed. Each time we encounter None in src_url_str or dst_url_str we # populate from the respective iterator, and we reset one or the other value # to None after yielding an action that disposes of that URL. while True: if src_url_str is None: if self.sorted_src_urls_it.IsEmpty(): out_of_src_items = True else: (src_url_str, src_size, src_time_created, src_atime, src_mtime, src_mode, src_uid, src_gid, src_crc32c, src_md5) = (self._ParseTmpFileLine(next(self.sorted_src_urls_it))) posix_attrs = POSIXAttributes(atime=src_atime, mtime=src_mtime, uid=src_uid, gid=src_gid, mode=src_mode) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). src_url_str_to_check = _EncodeUrl( src_url_str[base_src_url_len:].replace('\\', '/')) dst_url_str_would_copy_to = copy_helper.ConstructDstUrl( src_url=self.base_src_url, exp_src_url=StorageUrlFromString(src_url_str), src_url_names_container=True, have_multiple_srcs=True, has_multiple_top_level_srcs=False, exp_dst_url=self.base_dst_url, have_existing_dest_subdir=False, recursion_requested=self.recursion_requested).url_string if dst_url_str is None: if not self.sorted_dst_urls_it.IsEmpty(): # We don't need time created at the destination. (dst_url_str, dst_size, _, dst_atime, dst_mtime, dst_mode, dst_uid, dst_gid, dst_crc32c, dst_md5) = self._ParseTmpFileLine(next(self.sorted_dst_urls_it)) # Skip past base URL and normalize slashes so we can compare across # clouds/file systems (including Windows). dst_url_str_to_check = _EncodeUrl( dst_url_str[base_dst_url_len:].replace('\\', '/')) # Only break once we've attempted to populate {str,dst}_url_to_check and # we know we're out of src objects. if out_of_src_items: break # We're guaranteed to have a value for src_url_str_to_check here, but may # be out of dst objects. if (dst_url_str is None or src_url_str_to_check < dst_url_str_to_check): # There's no dst object corresponding to src object, so copy src to dst. yield RsyncDiffToApply(src_url_str, dst_url_str_would_copy_to, posix_attrs, DiffAction.COPY, src_size) src_url_str = None elif src_url_str_to_check > dst_url_str_to_check: # dst object without a corresponding src object, so remove dst if -d # option was specified. if self.delete_extras: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) dst_url_str = None else: # There is a dst object corresponding to src object, so check if objects # match. if (StorageUrlFromString(src_url_str).IsCloudUrl() and StorageUrlFromString(dst_url_str).IsFileUrl() and src_mtime == NA_TIME): src_mtime = src_time_created should_replace, has_src_mtime, has_dst_mtime = (self._CompareObjects( src_url_str, src_size, src_mtime, src_crc32c, src_md5, dst_url_str, dst_size, dst_mtime, dst_crc32c, dst_md5)) if should_replace: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.COPY, src_size) elif self.preserve_posix: posix_attrs, needs_update = NeedsPOSIXAttributeUpdate( src_atime, dst_atime, src_mtime, dst_mtime, src_uid, dst_uid, src_gid, dst_gid, src_mode, dst_mode) if needs_update: yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.POSIX_SRC_TO_DST, src_size) elif has_src_mtime and not has_dst_mtime: # File/object at destination matches source but is missing mtime # attribute at destination. yield RsyncDiffToApply(src_url_str, dst_url_str, posix_attrs, DiffAction.MTIME_SRC_TO_DST, src_size) # else: we don't need to copy the file from src to dst since they're # the same files. # Advance to the next two objects. src_url_str = None dst_url_str = None if not self.delete_extras: return # If -d option was specified any files/objects left in dst iteration should # be removed. if dst_url_str: yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) for line in self.sorted_dst_urls_it: (dst_url_str, _, _, _, _, _, _, _, _, _) = self._ParseTmpFileLine(line) yield RsyncDiffToApply(None, dst_url_str, POSIXAttributes(), DiffAction.REMOVE, None) class _SeekAheadDiffIterator(object): """Wraps _AvoidChecksumAndListingDiffIterator and yields SeekAheadResults.""" def __init__(self, cloned_diff_iterator): self.cloned_diff_iterator = cloned_diff_iterator def __iter__(self): for diff_to_apply in self.cloned_diff_iterator: bytes_to_copy = diff_to_apply.copy_size or 0 if (diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST or diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST): # Assume MTIME_SRC_TO_DST and POSIX_SRC_TO_DST are metadata-only # copies. However, if the user does not have OWNER permission on # an object, the data must be re-sent, and this function will # underestimate the amount of bytes that rsync must copy. bytes_to_copy = 0 yield SeekAheadResult(data_bytes=bytes_to_copy) class _AvoidChecksumAndListingDiffIterator(_DiffIterator): """Iterator initialized from an existing _DiffIterator. This iterator yields RsyncDiffToApply objects used to estimate the total work that will be performed by the DiffIterator, while avoiding expensive computation. """ # pylint: disable=super-init-not-called def __init__(self, initialized_diff_iterator): # Intentionally don't call the _DiffIterator constructor. This class # reuses the initialized_diff_iterator values to avoid unnecessary # computation, and inherits the __iter__ function. # We're providing an estimate, so avoid computing checksums even though # that may cause our estimate to be off. self.compute_file_checksums = False self.delete_extras = initialized_diff_iterator.delete_extras self.recursion_requested = initialized_diff_iterator.delete_extras # TODO: Add a test that mocks the appropriate values in RsyncFunc and # ensure that running this iterator succeeds. self.preserve_posix = False # This iterator shouldn't output any log messages. self.logger = logging.getLogger('dummy') self.base_src_url = initialized_diff_iterator.base_src_url self.base_dst_url = initialized_diff_iterator.base_dst_url self.skip_old_files = initialized_diff_iterator.skip_old_files self.ignore_existing = initialized_diff_iterator.ignore_existing # Note that while this leaves 2 open file handles, we track these in a # global list to be closed (if not closed in the calling scope) and deleted # at exit time. self.sorted_list_src_file = open( initialized_diff_iterator.sorted_list_src_file_name, 'r') self.sorted_list_dst_file = open( initialized_diff_iterator.sorted_list_dst_file_name, 'r') _tmp_files.append(self.sorted_list_src_file) _tmp_files.append(self.sorted_list_dst_file) # Wrap iterators in PluralityCheckableIterator so we can check emptiness. self.sorted_src_urls_it = PluralityCheckableIterator( iter(self.sorted_list_src_file)) self.sorted_dst_urls_it = PluralityCheckableIterator( iter(self.sorted_list_dst_file)) # pylint: enable=super-init-not-called def _RsyncFunc(cls, diff_to_apply, thread_state=None): """Worker function for performing the actual copy and remove operations.""" gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) dst_url_str = diff_to_apply.dst_url_str dst_url = StorageUrlFromString(dst_url_str) posix_attrs = diff_to_apply.src_posix_attrs if diff_to_apply.diff_action == DiffAction.REMOVE: if cls.dryrun: cls.logger.info('Would remove %s', dst_url) else: cls.logger.info('Removing %s', dst_url) if dst_url.IsFileUrl(): try: os.unlink(dst_url.object_name) except FileNotFoundError: # Missing file errors occur occasionally with .gstmp files # and can be ignored for deletes. cls.logger.debug('%s was already removed', dst_url) pass else: try: gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=dst_url.generation, provider=dst_url.scheme) except NotFoundException: # If the object happened to be deleted by an external process, this # is fine because it moves us closer to the desired state. pass elif diff_to_apply.diff_action == DiffAction.COPY: src_url_str = diff_to_apply.src_url_str src_url = StorageUrlFromString(src_url_str) if cls.dryrun: if src_url.IsFileUrl(): # Try to open the local file to detect errors that would occur in # non-dry-run mode. try: with open(src_url.object_name, 'rb') as _: pass except Exception as e: # pylint: disable=broad-except cls.logger.info('Could not open %s' % src_url.object_name) raise cls.logger.info('Would copy %s to %s', src_url, dst_url) else: try: src_obj_metadata = None if src_url.IsCloudUrl(): src_generation = GenerationFromUrlAndString(src_url, src_url.generation) src_obj_metadata = gsutil_api.GetObjectMetadata( src_url.bucket_name, src_url.object_name, generation=src_generation, provider=src_url.scheme, fields=cls.source_metadata_fields) if ObjectIsGzipEncoded(src_obj_metadata): cls.logger.info( '%s has a compressed content-encoding, so it will be ' 'decompressed upon download; future executions of gsutil rsync ' 'with this source object will always download it. If you wish ' 'to synchronize such an object efficiently, compress the ' 'source objects in place before synchronizing them, rather ' 'than (for example) using gsutil cp -Z to compress them ' 'on-the-fly (which results in compressed content-encoding).' % src_url) else: # src_url.IsFileUrl() src_obj_metadata = apitools_messages.Object() # getmtime can return a float, so it needs to be converted to long. if posix_attrs.mtime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('mtime', src_url.url_string) if src_url.IsFifo() or src_url.IsStream(): type_text = 'Streams' if src_url.IsStream() else 'Named pipes' cls.logger.warn( 'WARNING: %s are not supported by gsutil rsync and ' 'will likely fail. Use the -x option to exclude %s by name.', type_text, src_url.url_string) if src_obj_metadata.metadata: custom_metadata = src_obj_metadata.metadata else: custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata( posix_attrs, custom_metadata, preserve_posix=cls.preserve_posix_attrs) tmp_obj_metadata = apitools_messages.Object() tmp_obj_metadata.metadata = custom_metadata CopyCustomMetadata(tmp_obj_metadata, src_obj_metadata, override=True) copy_result = copy_helper.PerformCopy( cls.logger, src_url, dst_url, gsutil_api, cls, _RsyncExceptionHandler, src_obj_metadata=src_obj_metadata, headers=cls.headers, is_rsync=True, gzip_encoded=cls.gzip_encoded, gzip_exts=cls.gzip_exts, preserve_posix=cls.preserve_posix_attrs) if copy_result is not None: (_, bytes_transferred, _, _) = copy_result with cls.stats_lock: cls.total_bytes_transferred += bytes_transferred except SkipUnsupportedObjectError as e: cls.logger.info('Skipping item %s with unsupported object type %s', src_url, e.unsupported_type) elif diff_to_apply.diff_action == DiffAction.MTIME_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set mtime for %s', dst_url) else: cls.logger.info('Copying mtime from src to dst for %s', dst_url.url_string) mtime = posix_attrs.mtime obj_metadata = apitools_messages.Object() obj_metadata.metadata = CreateCustomMetadata({MTIME_ATTR: mtime}) if dst_url.IsCloudUrl(): dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) try: # Assume we have permission, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # We don't have permission to patch apparently, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: ParseAndSetPOSIXAttributes(dst_url.object_name, obj_metadata, preserve_posix=cls.preserve_posix_attrs) elif diff_to_apply.diff_action == DiffAction.POSIX_SRC_TO_DST: # If the destination is an object in a bucket, this will not blow away other # metadata. This behavior is unlike if the file/object actually needed to be # copied from the source to the destination. dst_url = StorageUrlFromString(diff_to_apply.dst_url_str) if cls.dryrun: cls.logger.info('Would set POSIX attributes for %s', dst_url) else: cls.logger.info('Copying POSIX attributes from src to dst for %s', dst_url.url_string) obj_metadata = apitools_messages.Object() obj_metadata.metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) SerializeFileAttributesToObjectMetadata(posix_attrs, obj_metadata.metadata, preserve_posix=True) if dst_url.IsCloudUrl(): dst_generation = GenerationFromUrlAndString(dst_url, dst_url.generation) dst_obj_metadata = gsutil_api.GetObjectMetadata( dst_url.bucket_name, dst_url.object_name, generation=dst_generation, provider=dst_url.scheme, fields=['acl']) try: # Assume we have ownership, and can patch the object. gsutil_api.PatchObjectMetadata(dst_url.bucket_name, dst_url.object_name, obj_metadata, provider=dst_url.scheme, generation=dst_url.generation) except ServiceException as err: cls.logger.debug('Error while trying to patch: %s', err) # Apparently we don't have object ownership, so it must be copied. cls.logger.info( 'Copying whole file/object for %s instead of patching' ' because you don\'t have patch permission on the ' 'object.', dst_url.url_string) _RsyncFunc(cls, RsyncDiffToApply(diff_to_apply.src_url_str, diff_to_apply.dst_url_str, posix_attrs, DiffAction.COPY, diff_to_apply.copy_size), thread_state=thread_state) else: raise CommandException('Got unexpected DiffAction (%d)' % diff_to_apply.diff_action) def _RootListingExceptionHandler(cls, e): """Simple exception handler for exceptions during listing URLs to sync.""" cls.logger.error(str(e)) def _RsyncExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while syncing:\n%s\n', traceback.format_exc()) class RsyncCommand(Command): """Implementation of gsutil rsync command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'rsync', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=2, max_args=2, supported_sub_args='a:cCdenpPriRuUx:y:j:J', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[CommandArgument.MakeNCloudOrFileURLsArgument(2)]) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='rsync', help_name_aliases=['sync', 'synchronize'], help_type='command_help', help_one_line_summary='Synchronize content of two buckets/directories', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): ShimTranslatePredefinedAclSubOptForCopy(self.sub_opts) gcloud_command = ['storage', 'rsync'] flag_keys = [flag for flag, _ in self.sub_opts] if '-e' not in flag_keys: gcloud_command += ['--no-ignore-symlinks'] self.logger.warn( 'By default, gsutil copies file symlinks, but, by default, this' ' command (run via the gcloud storage shim) does not copy any' ' symlinks.') if '-P' in flag_keys: _, (source_path, destination_path) = self.ParseSubOpts( should_update_sub_opts_and_args=False) if (StorageUrlFromString(source_path).IsCloudUrl() and StorageUrlFromString(destination_path).IsFileUrl()): self.logger.warn( 'For preserving POSIX with rsync downloads, gsutil aborts if a' ' single download will result in invalid destination POSIX.' ' However, this command (run via the gcloud storage shim) will' ' skip invalid copies and still perform valid copies.') gcloud_storage_map = GcloudStorageMap( gcloud_command=gcloud_command, flag_map={ '-a': GcloudStorageFlag('--predefined-acl'), '-c': GcloudStorageFlag('--checksums-only'), '-C': GcloudStorageFlag('--continue-on-error'), '-d': GcloudStorageFlag('--delete-unmatched-destination-objects'), '-e': GcloudStorageFlag('--ignore-symlinks'), '-i': GcloudStorageFlag('--no-clobber'), '-J': GcloudStorageFlag('--gzip-in-flight-all'), '-j': GcloudStorageFlag('--gzip-in-flight'), '-n': GcloudStorageFlag('--dry-run'), '-P': GcloudStorageFlag('--preserve-posix'), '-p': GcloudStorageFlag('--preserve-acl'), '-R': GcloudStorageFlag('--recursive'), '-r': GcloudStorageFlag('--recursive'), '-U': GcloudStorageFlag('--skip-unsupported'), '-u': GcloudStorageFlag('--skip-if-dest-has-newer-mtime'), '-x': GcloudStorageFlag('--exclude'), }, ) return super().get_gcloud_storage_args(gcloud_storage_map) def _InsistContainer(self, url_str, treat_nonexistent_object_as_subdir): """Sanity checks that URL names an existing container. Args: url_str: URL string to check. treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. Returns: URL for checked string. Raises: CommandException if url_str doesn't name an existing container. """ (url, have_existing_container) = copy_helper.ExpandUrlToSingleBlr( url_str, self.gsutil_api, self.project_id, treat_nonexistent_object_as_subdir, logger=self.logger) if not have_existing_container: raise CommandException( 'arg (%s) does not name a directory, bucket, or bucket subdir.\n' 'If there is an object with the same path, please add a trailing\n' 'slash to specify the directory.' % url_str) return url def RunCommand(self): """Command entry point for the rsync command.""" self._ParseOpts() self.total_bytes_transferred = 0 # Use a lock to ensure accurate statistics in the face of # multi-threading/multi-processing. self.stats_lock = parallelism_framework_util.CreateLock() if not UsingCrcmodExtension(): if self.compute_file_checksums: self.logger.warn(SLOW_CRCMOD_WARNING) else: self.logger.warn(SLOW_CRCMOD_RSYNC_WARNING) src_url = self._InsistContainer(self.args[0], False) dst_url = self._InsistContainer(self.args[1], True) is_daisy_chain = (src_url.IsCloudUrl() and dst_url.IsCloudUrl() and src_url.scheme != dst_url.scheme) LogPerformanceSummaryParams(has_file_src=src_url.IsFileUrl(), has_cloud_src=src_url.IsCloudUrl(), has_file_dst=dst_url.IsFileUrl(), has_cloud_dst=dst_url.IsCloudUrl(), is_daisy_chain=is_daisy_chain, uses_fan=self.parallel_operations, provider_types=[src_url.scheme, dst_url.scheme]) self.source_metadata_fields = GetSourceFieldsNeededForCopy( dst_url.IsCloudUrl(), self.skip_unsupported_objects, self.preserve_acl, is_rsync=True, preserve_posix=self.preserve_posix_attrs) # Tracks if any copy or rm operations failed. self.op_failure_count = 0 # Tuple of attributes to share/manage across multiple processes in # parallel (-m) mode. shared_attrs = ('op_failure_count', 'total_bytes_transferred') for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _HandleSignals) process_count, thread_count = self._GetProcessAndThreadCount( process_count=None, thread_count=None, parallel_operations_override=self.ParallelOverrideReason.SPEED, print_macos_warning=False) copy_helper.TriggerReauthForDestinationProviderIfNecessary( dst_url, self.gsutil_api, worker_count=process_count thread_count, ) # Perform sync requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. diff_iterator = _DiffIterator(self, src_url, dst_url) # For estimation purposes, create a SeekAheadIterator based on the # source and destination files generated when creating the diff iterator. # This iteration should avoid expensive operations like file checksumming. seek_ahead_iterator = _SeekAheadDiffIterator( _AvoidChecksumAndListingDiffIterator(diff_iterator)) self.logger.info('Starting synchronization...') start_time = time.time() try: self.Apply(_RsyncFunc, diff_iterator, _RsyncExceptionHandler, shared_attrs, arg_checker=_DiffToApplyArgChecker, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) finally: CleanUpTempFiles() end_time = time.time() self.total_elapsed_time = end_time - start_time self.total_bytes_per_second = CalculateThroughput( self.total_bytes_transferred, self.total_elapsed_time) LogPerformanceSummaryParams( avg_throughput=self.total_bytes_per_second, total_elapsed_time=self.total_elapsed_time, total_bytes_transferred=self.total_bytes_transferred) if self.op_failure_count: plural_str = 's' if self.op_failure_count else '' raise CommandException('%d file%s/object%s could not be copied/removed.' % (self.op_failure_count, plural_str, plural_str)) def _ParseOpts(self): # exclude_symlinks is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.exclude_symlinks = False # continue_on_error is handled by Command parent class, so save in Command # state rather than CopyHelperOpts. self.continue_on_error = False self.delete_extras = False self.preserve_acl = False self.preserve_posix_attrs = False self.compute_file_checksums = False self.dryrun = False self.exclude_dirs = False self.exclude_pattern = None self.skip_old_files = False self.ignore_existing = False self.skip_unsupported_objects = False # self.recursion_requested is initialized in command.py (so it can be # checked in parent class for all commands). canned_acl = None # canned_acl is handled by a helper function in parent # Command class, so save in Command state rather than CopyHelperOpts. self.canned = None # Files matching these extensions should be compressed. # The gzip_encoded flag marks if the files should be compressed during # the upload. gzip_encoded = False gzip_arg_exts = None gzip_arg_all = None if self.sub_opts: for o, a in self.sub_opts: if o == '-a': canned_acl = a self.canned = True if o == '-c': self.compute_file_checksums = True # Note: In gsutil cp command this is specified using -c but here we use # -C so we can use -c for checksum arg (to be consistent with Unix rsync # command options). elif o == '-C': self.continue_on_error = True elif o == '-d': self.delete_extras = True elif o == '-e': self.exclude_symlinks = True elif o == '-j': gzip_encoded = True gzip_arg_exts = [x.strip() for x in a.split(',')] elif o == '-J': gzip_encoded = True gzip_arg_all = GZIP_ALL_FILES elif o == '-n': self.dryrun = True elif o == '-p': self.preserve_acl = True elif o == '-P': self.preserve_posix_attrs = True if not IS_WINDOWS: InitializePreservePosixData() elif o == '-r' or o == '-R': self.recursion_requested = True elif o == '-u': self.skip_old_files = True elif o == '-i': self.ignore_existing = True elif o == '-U': self.skip_unsupported_objects = True elif o == '-x' or o == '-y': if o == '-y': self.exclude_dirs = True if not a: raise CommandException('Invalid blank exclude filter') try: self.exclude_pattern = re.compile(a) except re.error: raise CommandException('Invalid exclude filter (%s)' % a) if self.preserve_acl and canned_acl: raise CommandException( 'Specifying both the -p and -a options together is invalid.') if gzip_arg_exts and gzip_arg_all: raise CommandException( 'Specifying both the -j and -J options together is invalid.') self.gzip_encoded = gzip_encoded self.gzip_exts = gzip_arg_exts or gzip_arg_all return CreateCopyHelperOpts( canned_acl=canned_acl, preserve_acl=self.preserve_acl, skip_unsupported_objects=self.skip_unsupported_objects)	79.51 KB	2025-06-25 08:19
📄 setmeta.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of setmeta command for setting cloud object metadata.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from apitools.base.py import encoding from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import MetadataMessage from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.metadata_util import IsCustomMetadataHeader from gslib.utils.retry_util import Retry from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistAsciiHeader from gslib.utils.text_util import InsistAsciiHeaderValue from gslib.utils.translation_helper import CopyObjectMetadata from gslib.utils.translation_helper import ObjectMetadataFromHeaders from gslib.utils.translation_helper import PreconditionsFromHeaders _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout _SYNOPSIS = """ gsutil setmeta -h [header:value\|header] ... url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil setmeta command allows you to set or remove the metadata on one or more objects. It takes one or more header arguments followed by one or more URLs, where each header argument is in one of two forms: - If you specify ``header:value``, it sets the provided value for the given header on all applicable objects. - If you specify ``header`` (with no value), it removes the given header from all applicable objects. For example, the following command sets the ``Content-Type`` and ``Cache-Control`` headers while also removing the ``Content-Disposition`` header on the specified objects: gsutil setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/.html If you have a large number of objects to update you might want to use the gsutil -m option, to perform a parallel (multi-threaded/multi-processing) update: gsutil -m setmeta -h "Content-Type:text/html" \\ -h "Cache-Control:public, max-age=3600" \\ -h "Content-Disposition" gs://bucket/.html You can also use the setmeta command to set custom metadata on an object: gsutil setmeta -h "x-goog-meta-icecreamflavor:vanilla" gs://bucket/object Custom metadata is always prefixed in gsutil with ``x-goog-meta-``. This distinguishes it from standard request headers. Other tools that send and receive object metadata by using the request body do not use this prefix. While gsutil supports custom metadata with arbitrary Unicode values, note that when setting metadata using the XML API, which sends metadata as HTTP headers, Unicode characters are encoded using UTF-8, then url-encoded to ASCII. For example: gsutil setmeta -h "x-goog-meta-foo: ã" gs://bucket/object stores the custom metadata key-value pair of ``foo`` and ``%C3%A3``. Subsequently, running ``ls -L`` using the JSON API to list the object's metadata prints ``%C3%A3``, while ``ls -L`` using the XML API url-decodes this value automatically, printing the character ``ã``. The setmeta command reads each object's current generation and metageneration and uses those as preconditions unless they are otherwise specified by top-level arguments. For example, the following command sets the custom metadata ``icecreamflavor:vanilla`` if the current live object has a metageneration of 2: gsutil -h "x-goog-if-metageneration-match:2" setmeta -h "x-goog-meta-icecreamflavor:vanilla" See `Object metadata <https://cloud.google.com/storage/docs/metadata>`_ for more information about object metadata. <B>OPTIONS</B> -h Specifies a header:value to be added, or header to be removed, from each named object. """) # Setmeta assumes a header-like model which doesn't line up with the JSON way # of doing things. This list comes from functionality that was supported by # gsutil3 at the time gsutil4 was released. SETTABLE_FIELDS = [ 'cache-control', 'content-disposition', 'content-encoding', 'content-language', 'content-type', 'custom-time' ] _GCLOUD_OBJECTS_UPDATE_COMMAND = ['storage', 'objects', 'update'] def _SetMetadataExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(e) cls.everything_set_okay = False def _SetMetadataFuncWrapper(cls, name_expansion_result, thread_state=None): cls.SetMetadataFunc(name_expansion_result, thread_state=thread_state) class SetMetaCommand(Command): """Implementation of gsutil setmeta command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'setmeta', command_name_aliases=['setheader'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='h:rR', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='setmeta', help_name_aliases=['setheader'], help_type='command_help', help_one_line_summary='Set metadata on already uploaded objects', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=_GCLOUD_OBJECTS_UPDATE_COMMAND, flag_map={}, ) def get_gcloud_storage_args(self): recursive_flag = [] for o, _ in self.sub_opts: if o in ('-r', '-R'): recursive_flag = ['--recursive'] break clear_set, set_dict = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) # Handle translation through "gcloud_command" instead of "flag_map". self.sub_opts = [] clear_flags = self._translate_headers( {clear_key: None for clear_key in clear_set}, unset=True) set_flags = self._translate_headers(set_dict, unset=False) command = (_GCLOUD_OBJECTS_UPDATE_COMMAND + recursive_flag + clear_flags + set_flags) gcloud_storage_map = GcloudStorageMap( gcloud_command=command, flag_map={}, ) return super().get_gcloud_storage_args(gcloud_storage_map) def RunCommand(self): """Command entry point for the setmeta command.""" metadata_minus, metadata_plus = self._ParseMetadataHeaders( self._GetHeaderStringsFromSubOpts()) self.metadata_change = metadata_plus for header in metadata_minus: self.metadata_change[header] = '' if not self.metadata_change: raise CommandException( 'gsutil setmeta requires one or more headers to be provided with the' ' -h flag. See "gsutil help setmeta" for more information.') if len(self.args) == 1 and not self.recursion_requested: url = StorageUrlFromString(self.args[0]) if not (url.IsCloudUrl() and url.IsObject()): raise CommandException('URL (%s) must name an object' % self.args[0]) # Used to track if any objects' metadata failed to be set. self.everything_set_okay = True self.preconditions = PreconditionsFromHeaders(self.headers) name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, self.args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.parallel_operations, bucket_listing_fields=['generation', 'metadata', 'metageneration']) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), self.args, self.recursion_requested, all_versions=self.all_versions, project_id=self.project_id) try: # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(_SetMetadataFuncWrapper, name_expansion_iterator, _SetMetadataExceptionHandler, fail_on_error=True, seek_ahead_iterator=seek_ahead_iterator) except AccessDeniedException as e: if e.status == 403: self._WarnServiceAccounts() raise if not self.everything_set_okay: raise CommandException('Metadata for some objects could not be set.') return 0 @Retry(PreconditionException, tries=3, timeout_secs=1) def SetMetadataFunc(self, name_expansion_result, thread_state=None): """Sets metadata on an object. Args: name_expansion_result: NameExpansionResult describing target object. thread_state: gsutil Cloud API instance to use for the operation. """ gsutil_api = GetCloudApiInstance(self, thread_state=thread_state) exp_src_url = name_expansion_result.expanded_storage_url self.logger.info('Setting metadata on %s...', exp_src_url) cloud_obj_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) preconditions = Preconditions( gen_match=self.preconditions.gen_match, meta_gen_match=self.preconditions.meta_gen_match) if preconditions.gen_match is None: preconditions.gen_match = cloud_obj_metadata.generation if preconditions.meta_gen_match is None: preconditions.meta_gen_match = cloud_obj_metadata.metageneration # Patch handles the patch semantics for most metadata, but we need to # merge the custom metadata field manually. patch_obj_metadata = ObjectMetadataFromHeaders(self.metadata_change) api = gsutil_api.GetApiSelector(provider=exp_src_url.scheme) # For XML we only want to patch through custom metadata that has # changed. For JSON we need to build the complete set. if api == ApiSelector.XML: pass elif api == ApiSelector.JSON: CopyObjectMetadata(patch_obj_metadata, cloud_obj_metadata, override=True) patch_obj_metadata = cloud_obj_metadata # Patch body does not need the object generation and metageneration. patch_obj_metadata.generation = None patch_obj_metadata.metageneration = None gsutil_api.PatchObjectMetadata(exp_src_url.bucket_name, exp_src_url.object_name, patch_obj_metadata, generation=exp_src_url.generation, preconditions=preconditions, provider=exp_src_url.scheme, fields=['id']) _PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _GetHeaderStringsFromSubOpts(self): """Gets header values from after the "setmeta" part of the command. Example: $ gsutil -h not:parsed setmeta is:parsed gs://bucket/object -> ["is:parsed"] Returns: List[str]: Headers without the "-h" but not yet split on colons. Raises: CommandException Found canned ACL. """ if not self.sub_opts: return [] headers = [] for o, a in self.sub_opts: if o == '-h': if 'x-goog-acl' in a or 'x-amz-acl' in a: raise CommandException( 'gsutil setmeta no longer allows canned ACLs. Use gsutil acl ' 'set ... to set canned ACLs.') headers.append(a) return headers def _ParseMetadataHeaders(self, headers): """Validates and parses metadata changes from the headers argument. Args: headers: Header dict to validate and parse. Returns: (metadata_plus, metadata_minus): Tuple of header sets to add and remove. """ metadata_minus = set() cust_metadata_minus = set() metadata_plus = {} cust_metadata_plus = {} # Build a count of the keys encountered from each plus and minus arg so we # can check for dupe field specs. num_metadata_plus_elems = 0 num_cust_metadata_plus_elems = 0 num_metadata_minus_elems = 0 num_cust_metadata_minus_elems = 0 for md_arg in headers: # Use partition rather than split, as we should treat all characters past # the initial : as part of the header's value. parts = md_arg.partition(':') (header, _, value) = parts InsistAsciiHeader(header) # Translate headers to lowercase to match the casing assumed by our # sanity-checking operations. lowercase_header = header.lower() # This check is overly simple; it would be stronger to check, for each # URL argument, whether the header starts with the provider # metadata_prefix, but here we just parse the spec once, before # processing any of the URLs. This means we will not detect if the user # tries to set an x-goog-meta- field on an another provider's object, # for example. is_custom_meta = IsCustomMetadataHeader(lowercase_header) if not is_custom_meta and lowercase_header not in SETTABLE_FIELDS: raise CommandException( 'Invalid or disallowed header (%s).\nOnly these fields (plus ' 'x-goog-meta-* fields) can be set or unset:\n%s' % (header, sorted(list(SETTABLE_FIELDS)))) if value: if is_custom_meta: # Allow non-ASCII data for custom metadata fields. cust_metadata_plus[header] = value num_cust_metadata_plus_elems += 1 else: # Don't unicode encode other fields because that would perturb their # content (e.g., adding %2F's into the middle of a Cache-Control # value). InsistAsciiHeaderValue(header, value) value = str(value) metadata_plus[lowercase_header] = value num_metadata_plus_elems += 1 else: if is_custom_meta: cust_metadata_minus.add(header) num_cust_metadata_minus_elems += 1 else: metadata_minus.add(lowercase_header) num_metadata_minus_elems += 1 if (num_metadata_plus_elems != len(metadata_plus) or num_cust_metadata_plus_elems != len(cust_metadata_plus) or num_metadata_minus_elems != len(metadata_minus) or num_cust_metadata_minus_elems != len(cust_metadata_minus) or metadata_minus.intersection(set(metadata_plus.keys()))): raise CommandException('Each header must appear at most once.') metadata_plus.update(cust_metadata_plus) metadata_minus.update(cust_metadata_minus) return (metadata_minus, metadata_plus)	15.86 KB	2025-06-25 08:19
📄 signurl.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of URL Signing workflow. see: https://cloud.google.com/storage/docs/access-control#Signed-URLs) """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import calendar import copy from datetime import datetime from datetime import timedelta from datetime import timezone import getpass import json import re import sys import six from six.moves import urllib from apitools.base.py.exceptions import HttpError from apitools.base.py.http_wrapper import MakeRequest from apitools.base.py.http_wrapper import Request from boto import config from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils import constants from gslib.utils.boto_util import GetNewHttp from gslib.utils.shim_util import GcloudStorageMap, GcloudStorageFlag from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl, to_bytes try: # Check for openssl. # pylint: disable=C6204 from OpenSSL.crypto import FILETYPE_PEM from OpenSSL.crypto import load_privatekey from OpenSSL.crypto import sign from OpenSSL.crypto import PKey HAVE_OPENSSL = True except ImportError: load_privatekey = None sign = None HAVE_OPENSSL = False FILETYPE_PEM = None try: from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.hazmat.primitives.serialization import pkcs12 from cryptography.hazmat._oid import NameOID HAVE_CRYPTO = True except ImportError: HAVE_CRYPTO = False _CRYPTO_IMPORT_ERROR = "pyca/cryptography is not available. Either install it, or please consider using the .json keyfile" _AUTO_DETECT_REGION = 'auto' _MAX_EXPIRATION_TIME = timedelta(days=7) _MAX_EXPIRATION_TIME_WITH_MINUS_U = timedelta(hours=12) _SYNOPSIS = """ gsutil signurl [-c <content_type>] [-d <duration>] [-m <http_method>] \\ [-p <password>] [-r <region>] [-b <project>] (-u \| <private-key-file>) \\ (gs://<bucket_name> \| gs://<bucket_name>/<object_name>)... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The signurl command will generate a signed URL that embeds authentication data so the URL can be used by someone who does not have a Google account. Please see the `Signed URLs documentation <https://cloud.google.com/storage/docs/access-control/signed-urls>`_ for background about signed URLs. Multiple gs:// URLs may be provided and may contain wildcards. A signed URL will be produced for each provided URL, authorized for the specified HTTP method and valid for the given duration. NOTE: Unlike the gsutil ls command, the signurl command does not support operations on sub-directories. For example, unless you have an object named ``some-directory/`` stored inside the bucket ``some-bucket``, the following command returns an error: ``gsutil signurl <private-key-file> gs://some-bucket/some-directory/`` The signurl command uses the private key for a service account (the '<private-key-file>' argument) to generate the cryptographic signature for the generated URL. The private key file must be in PKCS12 or JSON format. If the private key is encrypted the signed URL command will prompt for the passphrase used to protect the private key file (default 'notasecret'). For more information regarding generating a private key for use with the signurl command please see the `Authentication documentation. <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`_ If you used `service account credentials <https://cloud.google.com/storage/docs/gsutil/addlhelp/CredentialTypesSupportingVariousUseCases#supported-credential-types_1>`_ for authentication, you can replace the <private-key-file> argument with the -u or --use-service-account option to use the system-managed private key directly. This avoids the need to store a private key file locally, but prior to using this flag you must `configure <https://cloud.google.com/sdk/gcloud/reference/auth/activate-service-account>`_ ``gcloud`` to use your service account credentials. <B>OPTIONS</B> -b <project> Allows you to specify a user project that will be billed for requests that use the signed URL. This is useful for generating presigned links for buckets that use requester pays. Note that it's not valid to specify both the ``-b`` and ``--use-service-account`` options together. -c Specifies the content type for which the signed URL is valid for. -d Specifies the duration that the signed URL should be valid for, default duration is 1 hour. Times may be specified with no suffix (default hours), or with s = seconds, m = minutes, h = hours, d = days. This option may be specified multiple times, in which case the duration the link remains valid is the sum of all the duration options. The max duration allowed is 7 days when ``private-key-file`` is used. The max duration allowed is 12 hours when -u option is used. This limitation exists because the system-managed key used to sign the URL may not remain valid after 12 hours. -m Specifies the HTTP method to be authorized for use with the signed URL, default is GET. You may also specify RESUMABLE to create a signed resumable upload start URL. When using a signed URL to start a resumable upload session, you will need to specify the 'x-goog-resumable:start' header in the request or else signature validation will fail. -p Specify the private key password instead of prompting. -r <region> Specifies the `region <https://cloud.google.com/storage/docs/locations>`_ in which the resources for which you are creating signed URLs are stored. Default value is 'auto' which will cause gsutil to fetch the region for the resource. When auto-detecting the region, the current gsutil user's credentials, not the credentials from the private-key-file, are used to fetch the bucket's metadata. This option must be specified and not 'auto' when generating a signed URL to create a bucket. -u Use service account credentials instead of a private key file to sign the URL. You can also use the ``--use-service-account`` option, which is equivalent to ``-u``. Note that both options have a maximum allowed duration of 12 hours for a valid link. <B>USAGE</B> Create a signed URL for downloading an object valid for 10 minutes: gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object> Create a signed URL, valid for one hour, for uploading a plain text file via HTTP PUT: gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to PUT to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \\ gs://<bucket>/<obj> To construct a signed URL that allows anyone in possession of the URL to POST a resumable upload to the specified bucket for one day, creating an object of Content-Type image/jpg, run: gsutil signurl -m RESUMABLE -d 1d -c image/jpg <private-key-file> \\ gs://bucket/<obj> """) def _NowUTC(): """Returns the current utc time as a datetime object.""" return datetime.now(tz=timezone.utc).replace(tzinfo=None) def _DurationToTimeDelta(duration): r"""Parses the given duration and returns an equivalent timedelta.""" match = re.match(r'^(\d+)([dDhHmMsS])?$', duration) if not match: raise CommandException('Unable to parse duration string') duration, modifier = match.groups('h') duration = int(duration) modifier = modifier.lower() if modifier == 'd': ret = timedelta(days=duration) elif modifier == 'h': ret = timedelta(hours=duration) elif modifier == 'm': ret = timedelta(minutes=duration) elif modifier == 's': ret = timedelta(seconds=duration) return ret def _GenSignedUrl(key, api, use_service_account, provider, client_id, method, duration, gcs_path, logger, region, content_type=None, billing_project=None, string_to_sign_debug=False, generation=None): """Construct a string to sign with the provided key. Args: key: The private key to use for signing the URL. api: The CloudApiDelegator instance use_service_account: If True, use the service account credentials instead of using the key file to sign the URL provider: Cloud storage provider to connect to. If not present, class-wide default is used. client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. gcs_path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. content_type: Optional Content-Type for the signed URL. HTTP requests using the URL must match this Content-Type. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. generation: If not None, specifies a version of an object for signing. Returns: The complete URL (string). """ gs_host = config.get('Credentials', 'gs_host', 'storage.googleapis.com') signed_headers = {'host': gs_host} if method == 'RESUMABLE': method = 'POST' signed_headers['x-goog-resumable'] = 'start' if not content_type: logger.warn('Warning: no Content-Type header was specified with the -c ' 'flag, so uploads to the resulting Signed URL must not ' 'specify a Content-Type.') if content_type: signed_headers['content-type'] = content_type if use_service_account: final_url = api.SignUrl(provider=provider, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) else: if six.PY2: digest = b'RSA-SHA256' else: # Your IDE may complain about this due to a bad docstring in pyOpenSsl: # https://github.com/pyca/pyopenssl/issues/741 digest = 'RSA-SHA256' string_to_sign, canonical_query_string = CreatePayload( client_id=client_id, method=method, duration=duration, path=gcs_path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, billing_project=billing_project, string_to_sign_debug=string_to_sign_debug) if isinstance(key, PKey): raw_signature = sign(key, string_to_sign, digest) else: if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) raw_signature = key.sign(to_bytes(string_to_sign), padding.PKCS1v15(), hashes.SHA256()) final_url = GetFinalUrl(raw_signature, gs_host, gcs_path, canonical_query_string) return final_url def _ReadKeystore(key_string, passwd): key_string = to_bytes(key_string) passwd = to_bytes(passwd) # Cryptography package is not bundled with gsutil, Keeping it at top would throw error. if not HAVE_CRYPTO: raise CommandException(_CRYPTO_IMPORT_ERROR) try: key, cert, add = pkcs12.load_key_and_certificates(key_string, passwd) client_email = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value return key, client_email except: raise CommandException('Unable to load the keyfile, Invalid password or PKCS12 data.') def _ReadJSONKeystore(ks_contents, passwd=None): """Read the client email and private key from a JSON keystore. Assumes this keystore was downloaded from the Cloud Platform Console. By default, JSON keystore private keys from the Cloud Platform Console aren't encrypted so the passwd is optional as load_privatekey will prompt for the PEM passphrase if the key is encrypted. Arguments: ks_contents: JSON formatted string representing the keystore contents. Must be a valid JSON string and contain the fields 'private_key' and 'client_email'. passwd: Passphrase for encrypted private keys. Returns: key: Parsed private key from the keystore. client_email: The email address for the service account. Raises: ValueError: If unable to parse ks_contents or keystore is missing required fields. """ # ensuring that json.loads receives unicode in Python 3 and bytes in Python 2 # Previous to Python 3.6, there was no automatic conversion and str was req. ks = json.loads(six.ensure_str(ks_contents)) if 'client_email' not in ks or 'private_key' not in ks: raise ValueError('JSON keystore doesn\'t contain required fields') client_email = ks['client_email'] if passwd: key = load_privatekey(FILETYPE_PEM, ks['private_key'], passwd) else: key = load_privatekey(FILETYPE_PEM, ks['private_key']) return key, client_email class UrlSignCommand(Command): """Implementation of gsutil url_sign command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'signurl', command_name_aliases=['signedurl', 'queryauth'], usage_synopsis=_SYNOPSIS, min_args=1, max_args=constants.NO_MAX, supported_sub_args='m:d:b:c:p:r:u', supported_private_args=['use-service-account'], file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreFileURLsArgument(), CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='signurl', help_name_aliases=[ 'signedurl', 'queryauth', ], help_type='command_help', help_one_line_summary='Create a signed URL', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def get_gcloud_storage_args(self): # TODO (b/287273841): Replace `copy` with a better pattern for # translating flag values. original_args = copy.deepcopy(self.args) original_sub_opts = copy.deepcopy(self.sub_opts) gcloud_command = [ 'storage', 'sign-url', '--format=csv[separator="\\t"](resource:label="URL",' ' http_verb:label="HTTP Method",' ' expiration:label="Expiration",' ' signed_url:label="Signed URL")', '--private-key-file=' + self.args[0] ] self.args = self.args[1:] duration_arg_idx = None http_verb_arg_idx = None content_type_arg_idx = None billing_project_arg_idx = None for i, (flag, _) in enumerate(self.sub_opts): if flag == '-d': duration_arg_idx = i elif flag == '-m': http_verb_arg_idx = i elif flag == '-c': content_type_arg_idx = i elif flag == '-b': billing_project_arg_idx = i if duration_arg_idx is not None: # Convert duration to seconds, which gcloud can handle. seconds = str( int( _DurationToTimeDelta( self.sub_opts[duration_arg_idx][1]).total_seconds())) + 's' self.sub_opts[duration_arg_idx] = ('-d', seconds) if http_verb_arg_idx is not None: if self.sub_opts[http_verb_arg_idx][1] == 'RESUMABLE': self.sub_opts[http_verb_arg_idx] = ('-m', 'POST') gcloud_command += ['--headers=x-goog-resumable=start'] if content_type_arg_idx is not None: content_type_value = self.sub_opts[content_type_arg_idx][1] self.sub_opts[content_type_arg_idx] = ('-c', 'content-type=' + content_type_value) if billing_project_arg_idx is not None: project_value = self.sub_opts[billing_project_arg_idx][1] self.sub_opts[billing_project_arg_idx] = ('-b', 'userProject=' + project_value) fully_translated_command = super().get_gcloud_storage_args( GcloudStorageMap(gcloud_command=gcloud_command, flag_map={ '-m': GcloudStorageFlag('--http-verb'), '-d': GcloudStorageFlag('--duration'), '-b': GcloudStorageFlag('--query-params'), '-c': GcloudStorageFlag('--headers'), '-r': GcloudStorageFlag('--region'), '-p': GcloudStorageFlag('--private-key-password'), })) # Ensures dry run mode works correctly, as flag translation requires # mutating command state. # TODO(b/287273841): Refactor so that there's a better pattern for # translating flag values. self.args = original_args self.sub_opts = original_sub_opts return fully_translated_command def _ParseAndCheckSubOpts(self): # Default argument values delta = None method = 'GET' content_type = '' passwd = None region = _AUTO_DETECT_REGION use_service_account = False billing_project = None for o, v in self.sub_opts: # TODO(PY3-ONLY): Delete this if block. if six.PY2: v = v.decode(sys.stdin.encoding or constants.UTF8) if o == '-d': if delta is not None: delta += _DurationToTimeDelta(v) else: delta = _DurationToTimeDelta(v) elif o == '-m': method = v elif o == '-c': content_type = v elif o == '-p': passwd = v elif o == '-r': region = v elif o == '-u' or o == '--use-service-account': use_service_account = True elif o == '-b': billing_project = v else: self.RaiseInvalidArgumentException() if delta is None: delta = timedelta(hours=1) else: if use_service_account and delta > _MAX_EXPIRATION_TIME_WITH_MINUS_U: # This restriction comes from the IAM SignBlob API. The SignBlob # API uses a system-managed key which can guarantee validation only # up to 12 hours. b/156160482#comment4 raise CommandException( 'Max valid duration allowed is %s when -u flag is used. For longer' ' duration, consider using the private-key-file instead of the -u' ' option.' % _MAX_EXPIRATION_TIME_WITH_MINUS_U) elif delta > _MAX_EXPIRATION_TIME: raise CommandException('Max valid duration allowed is ' '%s' % _MAX_EXPIRATION_TIME) if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']: raise CommandException('HTTP method must be one of' '[GET\|HEAD\|PUT\|DELETE\|RESUMABLE]') if not use_service_account and len(self.args) < 2: raise CommandException( 'The command requires a key file argument and one or more ' 'URL arguments if the --use-service-account flag is missing. ' 'Run `gsutil help signurl` for more info') if use_service_account and billing_project: raise CommandException( 'Specifying both the -b and --use-service-account options together is' 'invalid.') return method, delta, content_type, passwd, region, use_service_account, billing_project def _ProbeObjectAccessWithClient(self, key, use_service_account, provider, client_email, gcs_path, generation, logger, region, billing_project): """Performs a head request against a signed URL to check for read access.""" # Choose a reasonable time in the future; if the user's system clock is # 60 or more seconds behind the server's this will generate an error. signed_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=provider, client_id=client_email, method='HEAD', duration=timedelta(seconds=60), gcs_path=gcs_path, generation=generation, logger=logger, region=region, billing_project=billing_project, string_to_sign_debug=True) try: h = GetNewHttp() req = Request(signed_url, 'HEAD') response = MakeRequest(h, req) if response.status_code not in [200, 403, 404]: raise HttpError.FromResponse(response) return response.status_code except HttpError as http_error: if http_error.has_attr('response'): error_response = http_error.response error_string = ('Unexpected HTTP response code %s while querying ' 'object readability. Is your system clock accurate?' % error_response.status_code) if error_response.content: error_string += ' Content: %s' % error_response.content else: error_string = ('Expected an HTTP response code of ' '200 while querying object readability, but received ' 'an error: %s' % http_error) raise CommandException(error_string) def _EnumerateStorageUrls(self, in_urls): ret = [] for url_str in in_urls: if ContainsWildcard(url_str): ret.extend([blr.storage_url for blr in self.WildcardIterator(url_str)]) else: ret.append(StorageUrlFromString(url_str)) return ret def RunCommand(self): """Command entry point for signurl command.""" if not HAVE_OPENSSL: raise CommandException( 'The signurl command requires the pyopenssl library (try pip ' 'install pyopenssl or easy_install pyopenssl)') method, delta, content_type, passwd, region, use_service_account, billing_project = ( self._ParseAndCheckSubOpts()) arg_start_index = 0 if use_service_account else 1 storage_urls = self._EnumerateStorageUrls(self.args[arg_start_index:]) region_cache = {} key = None if not use_service_account: try: key, client_email = _ReadJSONKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: # Ignore and try parsing as a pkcs12. if not passwd: passwd = getpass.getpass('Keystore password:') try: key, client_email = _ReadKeystore( open(self.args[0], 'rb').read(), passwd) except ValueError: raise CommandException('Unable to parse private key from {0}'.format( self.args[0])) else: client_email = self.gsutil_api.GetServiceAccountId(provider='gs') print('URL\tHTTP Method\tExpiration\tSigned URL') for url in storage_urls: if url.scheme != 'gs': raise CommandException('Can only create signed urls from gs:// urls') if url.IsBucket(): if region == _AUTO_DETECT_REGION: raise CommandException('Generating signed URLs for creating buckets' ' requires a region be specified via the -r ' 'option. Run `gsutil help signurl` for more ' 'information about the \'-r\' option.') gcs_path = url.bucket_name if method == 'RESUMABLE': raise CommandException('Resumable signed URLs require an object ' 'name.') else: # Need to URL encode the object name as Google Cloud Storage does when # computing the string to sign when checking the signature. gcs_path = '{0}/{1}'.format( url.bucket_name, urllib.parse.quote(url.object_name.encode(constants.UTF8), safe=b'/~')) if region == _AUTO_DETECT_REGION: if url.bucket_name in region_cache: bucket_region = region_cache[url.bucket_name] else: try: _, bucket = self.GetSingleBucketUrlFromArg( 'gs://{}'.format(url.bucket_name), bucket_fields=['location']) except Exception as e: raise CommandException( '{}: Failed to auto-detect location for bucket \'{}\'. Please ' 'ensure you have storage.buckets.get permission on the bucket ' 'or specify the bucket\'s location using the \'-r\' option.'. format(e.__class__.__name__, url.bucket_name)) bucket_region = bucket.location.lower() region_cache[url.bucket_name] = bucket_region else: bucket_region = region final_url = _GenSignedUrl(key=key, api=self.gsutil_api, use_service_account=use_service_account, provider=url.scheme, client_id=client_email, method=method, duration=delta, gcs_path=gcs_path, generation=url.generation, logger=self.logger, region=bucket_region, content_type=content_type, billing_project=billing_project, string_to_sign_debug=True) expiration = calendar.timegm((datetime.now(tz=timezone.utc).replace(tzinfo=None) + delta).utctimetuple()) expiration_dt = datetime.fromtimestamp(expiration) time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S') # TODO(PY3-ONLY): Delete this if block. if six.PY2: time_str = time_str.decode(constants.UTF8) url_info_str = '{0}\t{1}\t{2}\t{3}'.format(url.url_string, method, time_str, final_url) # TODO(PY3-ONLY): Delete this if block. if six.PY2: url_info_str = url_info_str.encode(constants.UTF8) print(url_info_str) response_code = self._ProbeObjectAccessWithClient( key, use_service_account, url.scheme, client_email, gcs_path, url.generation, self.logger, bucket_region, billing_project) if response_code == 404: if url.IsBucket() and method != 'PUT': raise CommandException( 'Bucket {0} does not exist. Please create a bucket with ' 'that name before a creating signed URL to access it.'.format( url)) else: if method != 'PUT' and method != 'RESUMABLE': raise CommandException( 'Object {0} does not exist. Please create/upload an object ' 'with that name before a creating signed URL to access it.'. format(url)) elif response_code == 403: self.logger.warn( '%s does not have permissions on %s, using this link will likely ' 'result in a 403 error until at least READ permissions are granted', client_email or 'The account', url) return 0	28.89 KB	2025-06-25 08:19
📄 stat.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Unix-like stat command for cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import sys from gslib.bucket_listing_ref import BucketListingObject from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotFoundException from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import ContainsWildcard from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import NO_MAX from gslib.utils.ls_helper import ENCRYPTED_FIELDS from gslib.utils.ls_helper import PrintFullInfoAboutObject from gslib.utils.ls_helper import UNENCRYPTED_FULL_LISTING_FIELDS from gslib.utils.shim_util import GcloudStorageMap _SYNOPSIS = """ gsutil stat url... """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The stat command outputs `object metadata <https://cloud.google.com/storage/docs/metadata>`_ for the specified object URLs. It is similar to running: gsutil ls -L gs://some-bucket/some-object but is more efficient because it avoids listing objects and gets the minimum necessary amount of object metadata. Moreover, because it avoids listing objects (which for some storage providers are eventually consistent) the ``gsutil stat`` command provides a strongly consistent way to check for the existence (and read the metadata) of an object. The ``gsutil stat`` command does, however, make listing object requests if you specify URLs using wildcards. If run with the ``gsutil -q`` option nothing is printed, e.g.: gsutil -q stat gs://some-bucket/some-object This behavior can be useful when writing scripts: even though nothing is printed from the command, it still has an exit status of 0 for an existing object and 1 for a non-existent object. NOTE: Unlike the gsutil ls command, the stat command does not support operations on sub-directories. For example, if you run the command: gsutil -q stat gs://some-bucket/some-subdir/ gsutil looks for information about an object called "some-subdir/" (with a trailing slash) inside the bucket "some-bucket", as opposed to operating on objects nested under gs://some-bucket/some-subdir/. Unless you actually have an object with that name, the operation fails. However, you can use the stat command on objects within subdirectories. For example, this command works as expected: gsutil -q stat gs://some-bucket/some-subdir/file.txt """) # TODO: Add ability to stat buckets. class StatCommand(Command): """Implementation of gsutil stat command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'stat', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=1, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ ApiSelector.XML, ApiSelector.JSON, ], gs_default_api=ApiSelector.JSON, argparse_arguments=[ CommandArgument.MakeZeroOrMoreCloudURLsArgument(), ], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='stat', help_name_aliases=[], help_type='command_help', help_one_line_summary='Display object status', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) gcloud_storage_map = GcloudStorageMap( gcloud_command=[ # Formatting done in gcloud. 'storage', 'objects', 'list', '--fetch-encrypted-object-hashes', '--stat', ], flag_map={}, ) def RunCommand(self): """Command entry point for stat command.""" stat_fields = ENCRYPTED_FIELDS + UNENCRYPTED_FULL_LISTING_FIELDS found_nonmatching_arg = False for url_str in self.args: arg_matches = 0 url = StorageUrlFromString(url_str) if not url.IsObject(): raise CommandException('The stat command only works with object URLs') try: if ContainsWildcard(url_str): blr_iter = self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=stat_fields) else: try: single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=stat_fields) except EncryptionException: # Retry without requesting hashes. single_obj = self.gsutil_api.GetObjectMetadata( url.bucket_name, url.object_name, generation=url.generation, provider=url.scheme, fields=UNENCRYPTED_FULL_LISTING_FIELDS) blr_iter = [BucketListingObject(url, root_object=single_obj)] for blr in blr_iter: if blr.IsObject(): arg_matches += 1 # TODO: Request fewer fields if we're not printing the object. if logging.getLogger().isEnabledFor(logging.INFO): PrintFullInfoAboutObject(blr, incl_acl=False) except AccessDeniedException: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write('You aren\'t authorized to read %s - skipping' % url_str + '\n') except InvalidUrlError: raise except NotFoundException: pass if not arg_matches: if logging.getLogger().isEnabledFor(logging.INFO): sys.stderr.write(NO_URLS_MATCHED_TARGET % url_str + '\n') found_nonmatching_arg = True if found_nonmatching_arg: return 1 return 0	6.63 KB	2025-06-25 08:19
📄 test.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil test command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple import logging import os import subprocess import re import sys import tempfile import textwrap import time import traceback import six from six.moves import range import gslib from gslib.cloud_api import ProjectIdException from gslib.command import Command from gslib.command import ResetFailureCount from gslib.exception import CommandException from gslib.project_id import PopulateProjectId import gslib.tests as tests from gslib.tests.util import GetTestNames from gslib.tests.util import InvokedFromParFile from gslib.tests.util import unittest from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS # pylint: disable=g-import-not-at-top try: import coverage except ImportError: coverage = None if six.PY3: long = int _DEFAULT_TEST_PARALLEL_PROCESSES = 5 _DEFAULT_S3_TEST_PARALLEL_PROCESSES = 50 _SEQUENTIAL_ISOLATION_FLAG = 'sequential_only' _SYNOPSIS = """ gsutil test [-l] [-u] [-f] [command command...] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> The gsutil test command runs the gsutil unit tests and integration tests. The unit tests use an in-memory mock storage service implementation, while the integration tests send requests to the production service using the `preferred API <https://cloud.google.com/storage/docs/request-endpoints#gsutil>`_ set in the boto configuration file. CAUTION: The ``test`` command creates test buckets and objects in your project. Force quitting the ``test`` command can leave behind stale buckets, objects, and HMAC keys in your project. To run both the unit tests and integration tests, run the command with no arguments: gsutil test To run the unit tests only (which run quickly): gsutil test -u Tests run in parallel regardless of whether the top-level -m flag is present. To limit the number of tests run in parallel to 10 at a time: gsutil test -p 10 To force tests to run sequentially: gsutil test -p 1 To have sequentially-run tests stop running immediately when an error occurs: gsutil test -f To run tests for one or more individual commands add those commands as arguments. For example, the following command will run the cp and mv command tests: gsutil test cp mv To list available tests, run the test command with the -l argument: gsutil test -l The tests are defined in the code under the gslib/tests module. Each test file is of the format test_[name].py where [name] is the test name you can pass to this command. For example, running "gsutil test ls" would run the tests in "gslib/tests/test_ls.py". You can also run an individual test class or function name by passing the test module followed by the class name and optionally a test name. For example, to run the an entire test class by name: gsutil test naming.GsutilNamingTests or an individual test function: gsutil test cp.TestCp.test_streaming You can list the available tests under a module or class by passing arguments with the -l option. For example, to list all available test functions in the cp module: gsutil test -l cp To output test coverage: gsutil test -c -p 500 coverage html This will output an HTML report to a directory named 'htmlcov'. Test coverage is compatible with v4.1 of the coverage module (https://pypi.python.org/pypi/coverage). <B>OPTIONS</B> -b Run tests against multi-regional US buckets. By default, tests run against regional buckets in us-central1. -c Output coverage information. -f Exit on first sequential test failure. -l List available tests. -p N Run at most N tests in parallel. The default value is %d. -s Run tests against S3 instead of GS. -u Only run unit tests. """ % _DEFAULT_TEST_PARALLEL_PROCESSES) TestProcessData = namedtuple('TestProcessData', 'name return_code stdout stderr') def MakeCustomTestResultClass(total_tests): """Creates a closure of CustomTestResult. Args: total_tests: The total number of tests being run. Returns: An instance of CustomTestResult. """ class CustomTestResult(unittest.TextTestResult): """A subclass of unittest.TextTestResult that prints a progress report.""" def startTest(self, test): super(CustomTestResult, self).startTest(test) if self.dots: test_id = '.'.join(test.id().split('.')[-2:]) message = ('\r%d/%d finished - E[%d] F[%d] s[%d] - %s' % (self.testsRun, total_tests, len(self.errors), len(self.failures), len(self.skipped), test_id)) message = message[:73] message = message.ljust(73) self.stream.write('%s - ' % message) return CustomTestResult def GetTestNamesFromSuites(test_suite): """Takes a list of test suites and returns a list of contained test names.""" suites = [test_suite] test_names = [] while suites: suite = suites.pop() for test in suite: if isinstance(test, unittest.TestSuite): suites.append(test) else: test_names.append(test.id()[len('gslib.tests.test_'):]) return test_names # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def TestCaseToName(test_case): """Converts a python.unittest to its gsutil test-callable name.""" return (str(test_case.__class__).split('\'')[1] + '.' + test_case._testMethodName) # pylint: disable=protected-access # Need to get into the guts of unittest to evaluate test cases for parallelism. def SplitParallelizableTestSuite(test_suite): """Splits a test suite into groups with different running properties. Args: test_suite: A python unittest test suite. Returns: 4-part tuple of lists of test names: (tests that must be run sequentially, tests that must be isolated in a separate process but can be run either sequentially or in parallel, unit tests that can be run in parallel, integration tests that can run in parallel) """ # pylint: disable=import-not-at-top # Need to import this after test globals are set so that skip functions work. from gslib.tests.testcase.unit_testcase import GsUtilUnitTestCase isolated_tests = [] sequential_tests = [] parallelizable_integration_tests = [] parallelizable_unit_tests = [] items_to_evaluate = [test_suite] cases_to_evaluate = [] # Expand the test suites into individual test cases: while items_to_evaluate: suite_or_case = items_to_evaluate.pop() if isinstance(suite_or_case, unittest.suite.TestSuite): for item in suite_or_case._tests: items_to_evaluate.append(item) elif isinstance(suite_or_case, unittest.TestCase): cases_to_evaluate.append(suite_or_case) for test_case in cases_to_evaluate: test_method = getattr(test_case, test_case._testMethodName, None) if getattr(test_method, 'requires_isolation', False): # Test must be isolated to a separate process, even it if is being # run sequentially. isolated_tests.append(TestCaseToName(test_case)) elif not getattr(test_method, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif not getattr(test_case, 'is_parallelizable', True): sequential_tests.append(TestCaseToName(test_case)) elif isinstance(test_case, GsUtilUnitTestCase): parallelizable_unit_tests.append(TestCaseToName(test_case)) else: parallelizable_integration_tests.append(TestCaseToName(test_case)) return (sorted(sequential_tests), sorted(isolated_tests), sorted(parallelizable_unit_tests), sorted(parallelizable_integration_tests)) def CountFalseInList(input_list): """Counts number of falses in the input list.""" num_false = 0 for item in input_list: if not item: num_false += 1 return num_false def CreateTestProcesses(parallel_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=None): """Creates test processes to run tests in parallel. Args: parallel_tests: List of all parallel tests. test_index: List index of last created test before this function call. process_list: List of running subprocesses. Created processes are appended to this list. process_done: List of booleans indicating process completion. One 'False' will be added per process created. max_parallel_tests: Maximum number of tests to run in parallel. root_coverage_file: The root .coverage filename if coverage is requested. Returns: Index of last created test. """ orig_test_index = test_index # checking to see if test was invoked from a par file (bundled archive) # if not, add python executable path to ensure correct version of python # is used for testing executable_prefix = [sys.executable] if not InvokedFromParFile() else [] s3_argument = ['-s'] if tests.util.RUN_S3_TESTS else [] multiregional_buckets = ['-b'] if tests.util.USE_MULTIREGIONAL_BUCKETS else [] project_id_arg = [] try: project_id_arg = [ '-o', 'GSUtil:default_project_id=%s' % PopulateProjectId() ] except ProjectIdException: # If we don't have a project ID, unit tests should still be able to pass. pass process_create_start_time = time.time() last_log_time = process_create_start_time while (CountFalseInList(process_done) < max_parallel_tests and test_index < len(parallel_tests)): env = os.environ.copy() if root_coverage_file: env['GSUTIL_COVERAGE_OUTPUT_FILE'] = root_coverage_file envstr = dict() # constructing command list and ensuring each part is str cmd = [ six.ensure_str(part) for part in list( executable_prefix + [gslib.GSUTIL_PATH] + project_id_arg + ['test'] + s3_argument + multiregional_buckets + ['--' + _SEQUENTIAL_ISOLATION_FLAG] + [parallel_tests[test_index][len('gslib.tests.test_'):]] ) ] # yapf: disable for k, v in six.iteritems(env): envstr[six.ensure_str(k)] = six.ensure_str(v) process_list.append( subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=envstr)) test_index += 1 process_done.append(False) if time.time() - last_log_time > 5: print(('Created %d new processes (total %d/%d created)' % (test_index - orig_test_index, len(process_list), len(parallel_tests)))) last_log_time = time.time() if test_index == len(parallel_tests): print(('Test process creation finished (%d/%d created)' % (len(process_list), len(parallel_tests)))) return test_index class TestCommand(Command): """Implementation of gsutil test command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'test', command_name_aliases=[], usage_synopsis=_SYNOPSIS, min_args=0, max_args=NO_MAX, supported_sub_args='buflp:sc', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, supported_private_args=[_SEQUENTIAL_ISOLATION_FLAG], ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='test', help_name_aliases=[], help_type='command_help', help_one_line_summary=( 'Run gsutil unit/integration tests (for developers)'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunParallelTests(self, parallel_integration_tests, max_parallel_tests, coverage_filename): """Executes the parallel/isolated portion of the test suite. Args: parallel_integration_tests: List of tests to execute. max_parallel_tests: Maximum number of parallel tests to run at once. coverage_filename: If not None, filename for coverage output. Returns: (int number of test failures, float elapsed time) """ process_list = [] process_done = [] process_results = [] # Tuples of (name, return code, stdout, stderr) num_parallel_failures = 0 # Number of logging cycles we ran with no progress. progress_less_logging_cycles = 0 completed_as_of_last_log = 0 num_parallel_tests = len(parallel_integration_tests) parallel_start_time = last_log_time = time.time() test_index = CreateTestProcesses(parallel_integration_tests, 0, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) while len(process_results) < num_parallel_tests: for proc_num in range(len(process_list)): if process_done[proc_num] or process_list[proc_num].poll() is None: continue process_done[proc_num] = True stdout, stderr = process_list[proc_num].communicate() process_list[proc_num].stdout.close() process_list[proc_num].stderr.close() # TODO: Differentiate test failures from errors. if process_list[proc_num].returncode != 0: num_parallel_failures += 1 process_results.append( TestProcessData(name=parallel_integration_tests[proc_num], return_code=process_list[proc_num].returncode, stdout=stdout, stderr=stderr)) if len(process_list) < num_parallel_tests: test_index = CreateTestProcesses(parallel_integration_tests, test_index, process_list, process_done, max_parallel_tests, root_coverage_file=coverage_filename) if len(process_results) < num_parallel_tests: if time.time() - last_log_time > 5: print( '%d/%d finished - %d failures' % (len(process_results), num_parallel_tests, num_parallel_failures)) if len(process_results) == completed_as_of_last_log: progress_less_logging_cycles += 1 else: completed_as_of_last_log = len(process_results) # A process completed, so we made progress. progress_less_logging_cycles = 0 if progress_less_logging_cycles > 4: # Ran 5 or more logging cycles with no progress, let the user # know which tests are running slowly or hanging. still_running = [] for proc_num in range(len(process_list)): if not process_done[proc_num]: still_running.append(parallel_integration_tests[proc_num]) elapsed = time.time() - parallel_start_time print(('{sec} seconds elapsed since beginning parallel tests.\n' 'Still running: {procs}').format( sec=str(int(elapsed)), procs=still_running, )) # TODO: Terminate still-running processes if they # hang for a long time. last_log_time = time.time() time.sleep(1) process_run_finish_time = time.time() if num_parallel_failures: for result in process_results: if result.return_code != 0: new_stderr = result.stderr.split(b'\n') print('Results for failed test %s:' % result.name) for line in new_stderr: print(line.decode(UTF8).strip()) return (num_parallel_failures, (process_run_finish_time - parallel_start_time)) def PrintTestResults(self, num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed): """Prints test results for parallel and sequential tests.""" # TODO: Properly track test skips. print('Parallel tests complete. Success: %s Fail: %s' % (num_parallel_tests - num_parallel_failures, num_parallel_failures)) print(( 'Ran %d tests in %.3fs (%d sequential in %.3fs, %d parallel in %.3fs)' % (num_parallel_tests + num_sequential_tests, float(sequential_time_elapsed + parallel_time_elapsed), num_sequential_tests, float(sequential_time_elapsed), num_parallel_tests, float(parallel_time_elapsed)))) self.PrintSkippedTests(sequential_skipped) print() if not num_parallel_failures and sequential_success: print('OK') else: if num_parallel_failures: print('FAILED (parallel tests)') if not sequential_success: print('FAILED (sequential tests)') # TODO: Parallel skipped tests are never gathered anywhere, this needs implementation in RunParallelTests def PrintSkippedTests(self, sequential_skipped=set(), parallel_skipped=set()): """Prints all skipped tests, and the reasons they were skipped. Takes the union of sequentual_skipped and parallel_skipped, and pretty-prints the resulting methods and reasons. Note that these two arguments are lists of tuples from TestResult.skipped as described here: https://docs.python.org/2/library/unittest.html#unittest.TestResult.skipped Args: sequentual_skipped: An instance of TestResult.skipped. parallel_skipped: An instance of TestResult.skipped. """ if len(sequential_skipped) > 0 or len(parallel_skipped) > 0: sequential_skipped = set(sequential_skipped) parallel_skipped = set(parallel_skipped) all_skipped = sequential_skipped.union(parallel_skipped) print('Tests skipped:') for method, reason in all_skipped: print(' ' + method.id()) print(' Reason: ' + reason) def RunCommand(self): """Command entry point for the test command.""" failfast = False list_tests = False max_parallel_tests = _DEFAULT_TEST_PARALLEL_PROCESSES perform_coverage = False sequential_only = False if self.sub_opts: for o, a in self.sub_opts: if o == '-b': tests.util.USE_MULTIREGIONAL_BUCKETS = True elif o == '-c': perform_coverage = True elif o == '-f': failfast = True elif o == '-l': list_tests = True elif o == ('--' + _SEQUENTIAL_ISOLATION_FLAG): # Called to isolate a single test in a separate process. # Don't try to isolate it again (would lead to an infinite loop). sequential_only = True elif o == '-p': max_parallel_tests = long(a) elif o == '-s': if not tests.util.HAS_S3_CREDS: raise CommandException('S3 tests require S3 credentials. Please ' 'add appropriate credentials to your .boto ' 'file and re-run.') tests.util.RUN_S3_TESTS = True elif o == '-u': tests.util.RUN_INTEGRATION_TESTS = False if perform_coverage and not coverage: raise CommandException( 'Coverage has been requested but the coverage module was not found. ' 'You can install it with "pip install coverage".') if (tests.util.RUN_S3_TESTS and max_parallel_tests > _DEFAULT_S3_TEST_PARALLEL_PROCESSES): self.logger.warn( 'Reducing parallel tests to %d due to S3 maximum bucket ' 'limitations.', _DEFAULT_S3_TEST_PARALLEL_PROCESSES) max_parallel_tests = _DEFAULT_S3_TEST_PARALLEL_PROCESSES test_names = sorted(GetTestNames()) if list_tests and not self.args: print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 # Set list of commands to test if supplied. if self.args: commands_to_test = [] for name in self.args: if name in test_names or name.split('.')[0] in test_names: commands_to_test.append('gslib.tests.test_%s' % name) else: commands_to_test.append(name) else: commands_to_test = ['gslib.tests.test_%s' % name for name in test_names] # Installs a ctrl-c handler that tries to cleanly tear down tests. unittest.installHandler() loader = unittest.TestLoader() if commands_to_test: suite = unittest.TestSuite() for command_name in commands_to_test: try: suite_for_current_command = loader.loadTestsFromName(command_name) suite.addTests(suite_for_current_command) except (ImportError, AttributeError) as e: msg = ('Failed to import test code from file %s. TestLoader provided ' 'this error:\n\n%s' % (command_name, str(e))) # Try to give a better error message; by default, unittest swallows # ImportErrors and only shows that an import failed, not why. E.g.: # "'module' object has no attribute 'test_cp' try: __import__(command_name) except Exception as e: stack_trace = traceback.format_exc() err = re.sub('\\n', '\n ', stack_trace) msg += '\n\nAdditional traceback:\n\n%s' % (err) raise CommandException(msg) if list_tests: test_names = GetTestNamesFromSuites(suite) print('Found %d test names:' % len(test_names)) print(' ', '\n '.join(sorted(test_names))) return 0 if logging.getLogger().getEffectiveLevel() <= logging.INFO: verbosity = 1 else: verbosity = 2 logging.disable(logging.ERROR) if perform_coverage: # We want to run coverage over the gslib module, but filter out the test # modules and any third-party code. We also filter out anything under the # temporary directory. Otherwise, the gsutil update test (which copies # code to the temporary directory) gets included in the output. coverage_controller = coverage.coverage(source=['gslib'], omit=[ 'gslib/third_party/', 'gslib/tests/', tempfile.gettempdir() + '*', ]) coverage_controller.erase() coverage_controller.start() num_parallel_failures = 0 sequential_success = False (sequential_tests, isolated_tests, parallel_unit_tests, parallel_integration_tests) = (SplitParallelizableTestSuite(suite)) # Since parallel integration tests are run in a separate process, they # won't get the override to tests.util, so skip them here. if not tests.util.RUN_INTEGRATION_TESTS: parallel_integration_tests = [] logging.debug('Sequential tests to run: %s', sequential_tests) logging.debug('Isolated tests to run: %s', isolated_tests) logging.debug('Parallel unit tests to run: %s', parallel_unit_tests) logging.debug('Parallel integration tests to run: %s', parallel_integration_tests) # If we're running an already-isolated test (spawned in isolation by a # previous test process), or we have no parallel tests to run, # just run sequentially. For now, unit tests are always run sequentially. run_tests_sequentially = (sequential_only or (len(parallel_integration_tests) <= 1 and not isolated_tests)) # Disable analytics for the duration of testing. This is set as an # environment variable so that the subprocesses will also not report. os.environ['GSUTIL_TEST_ANALYTICS'] = '1' if run_tests_sequentially: total_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(total_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() else: if max_parallel_tests == 1: # We can't take advantage of parallelism, though we may have tests that # need isolation. sequential_tests += parallel_integration_tests parallel_integration_tests = [] sequential_start_time = time.time() # TODO: For now, run unit tests sequentially because they are fast. # We could potentially shave off several seconds of execution time # by executing them in parallel with the integration tests. if len(sequential_tests) + len(parallel_unit_tests): print('Running %d tests sequentially.' % (len(sequential_tests) + len(parallel_unit_tests))) sequential_tests_to_run = sequential_tests + parallel_unit_tests suite = loader.loadTestsFromNames( sorted([test_name for test_name in sequential_tests_to_run])) num_sequential_tests = suite.countTestCases() resultclass = MakeCustomTestResultClass(num_sequential_tests) runner = unittest.TextTestRunner(verbosity=verbosity, resultclass=resultclass, failfast=failfast) ret = runner.run(suite) sequential_success = ret.wasSuccessful() sequential_skipped = ret.skipped else: num_sequential_tests = 0 sequential_success = True sequential_time_elapsed = time.time() - sequential_start_time # At this point, all tests get their own process so just treat the # isolated tests as parallel tests. parallel_integration_tests += isolated_tests num_parallel_tests = len(parallel_integration_tests) if not num_parallel_tests: pass else: sequential_skipped = [] num_processes = min(max_parallel_tests, num_parallel_tests) if num_parallel_tests > 1 and max_parallel_tests > 1: message = 'Running %d tests in parallel mode (%d processes).' if num_processes > _DEFAULT_TEST_PARALLEL_PROCESSES: message += ( ' Please be patient while your CPU is incinerated. ' 'If your machine becomes unresponsive, consider reducing ' 'the amount of parallel test processes by running ' '\'gsutil test -p <num_processes>\'.') print(('\n'.join( textwrap.wrap(message % (num_parallel_tests, num_processes))))) else: print(('Running %d tests sequentially in isolated processes.' % num_parallel_tests)) (num_parallel_failures, parallel_time_elapsed) = self.RunParallelTests( parallel_integration_tests, max_parallel_tests, coverage_controller.data_files.filename if perform_coverage else None) self.PrintTestResults(num_sequential_tests, sequential_success, sequential_skipped, sequential_time_elapsed, num_parallel_tests, num_parallel_failures, parallel_time_elapsed) if perform_coverage: coverage_controller.stop() coverage_controller.combine() coverage_controller.save() print(('Coverage information was saved to: %s' % coverage_controller.data_files.filename)) # Re-enable analytics to report the test command. os.environ['GSUTIL_TEST_ANALYTICS'] = '0' if sequential_success and not num_parallel_failures: ResetFailureCount() return 0 return 1	28.28 KB	2025-06-25 08:19
📄 ubla.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module provides the command to gsutil.""" from __future__ import absolute_import from __future__ import print_function import getopt import textwrap from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils.text_util import InsistOnOrOff from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil ubla set (on\|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil ubla get bucket_url... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``ubla set`` command enables or disables uniform bucket-level access for Google Cloud Storage buckets. <B>SET EXAMPLES</B> Configure your buckets to use uniform bucket-level access: gsutil ubla set on gs://redbucket gs://bluebucket Configure your buckets to NOT use uniform bucket-level access: gsutil ubla set off gs://redbucket gs://bluebucket """ _GET_DESCRIPTION = """ <B>GET</B> The ``ubla get`` command shows whether uniform bucket-level access is enabled for the specified Cloud Storage bucket(s). <B>GET EXAMPLES</B> Check if your buckets are using uniform bucket-level access: gsutil ubla get gs://redbucket gs://bluebucket """ _DESCRIPTION = """ The ``ubla`` command is used to retrieve or configure the `uniform bucket-level access <https://cloud.google.com/storage/docs/bucket-policy-only>`_ setting of Cloud Storage bucket(s). This command has two sub-commands, ``get`` and ``set``. """ + _GET_DESCRIPTION + _SET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) # Aliases to make these more likely to fit on one line. IamConfigurationValue = apitools_messages.Bucket.IamConfigurationValue uniformBucketLevelAccessValue = IamConfigurationValue.BucketPolicyOnlyValue _GCLOUD_FORMAT_STRING = ( '--format=' + 'multi[terminator="' + shim_util.get_format_flag_newline() + '"](name:format="value(format(\'Uniform bucket-level' + ' access setting for gs://{}:\'))",' + ' iamConfiguration.uniformBucketLevelAccess.enabled.yesno(no="False")' + ':format="value[terminator=\'' + shim_util.get_format_flag_newline() + '\'](format(\' Enabled: {}\'))",' + ' iamConfiguration.uniformBucketLevelAccess.lockedTime.sub("T", " ")' + ':format="value(format(\' LockedTime: {}\'))")') class UblaCommand(Command): """Implements the gsutil ubla command.""" command_spec = Command.CreateCommandSpec( 'ubla', command_name_aliases=['uniformbucketlevelaccess'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'get': [CommandArgument.MakeNCloudURLsArgument(1),], 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], }) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='ubla', help_name_aliases=['uniformbucketlevelaccess'], help_type='command_help', help_one_line_summary='Configure Uniform bucket-level access', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', _GCLOUD_FORMAT_STRING, '--raw' ], flag_map={}, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--uniform-bucket-level-access', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-uniform-bucket-level-access', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _ValidateBucketListingRefAndReturnBucketName(self, blr): if blr.storage_url.scheme != 'gs': raise CommandException( 'The %s command can only be used with gs:// bucket URLs.' % self.command_name) def _GetUbla(self, blr): """Gets the Uniform bucket-level access setting for a bucket.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url bucket_metadata = self.gsutil_api.GetBucket(bucket_url.bucket_name, fields=['iamConfiguration'], provider=bucket_url.scheme) iam_config = bucket_metadata.iamConfiguration # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. uniform_bucket_level_access = iam_config.bucketPolicyOnly fields = { 'bucket': str(bucket_url).rstrip('/'), 'enabled': uniform_bucket_level_access.enabled } locked_time_line = '' if uniform_bucket_level_access.lockedTime: fields['locked_time'] = uniform_bucket_level_access.lockedTime locked_time_line = ' LockedTime: {locked_time}\n' if uniform_bucket_level_access: print(('Uniform bucket-level access setting for {bucket}:\n' ' Enabled: {enabled}\n' + locked_time_line).format(*fields)) def _SetUbla(self, blr, setting_arg): """Sets the Uniform bucket-level access setting for a bucket on or off.""" self._ValidateBucketListingRefAndReturnBucketName(blr) bucket_url = blr.storage_url iam_config = IamConfigurationValue() # TODO(mynameisrafe): Replace bucketPolicyOnly with uniformBucketLevelAccess # when the property is live. iam_config.bucketPolicyOnly = uniformBucketLevelAccessValue() iam_config.bucketPolicyOnly.enabled = (setting_arg == 'on') bucket_metadata = apitools_messages.Bucket(iamConfiguration=iam_config) setting_verb = 'Enabling' if setting_arg == 'on' else 'Disabling' print('%s Uniform bucket-level access for %s...' % (setting_verb, str(bucket_url).rstrip('/'))) self.gsutil_api.PatchBucket(bucket_url.bucket_name, bucket_metadata, fields=['iamConfiguration'], provider=bucket_url.scheme) return 0 def _Ubla(self): """Handles ubla command on a Cloud Storage bucket.""" subcommand = self.args.pop(0) if subcommand not in ('get', 'set'): raise CommandException('ubla only supports get\|set') subcommand_func = None subcommand_args = [] setting_arg = None if subcommand == 'get': subcommand_func = self._GetUbla elif subcommand == 'set': subcommand_func = self._SetUbla setting_arg = self.args.pop(0) InsistOnOrOff(setting_arg, 'Only on and off values allowed for set option') subcommand_args.append(setting_arg) # Iterate over bucket args, performing the specified subsubcommand. some_matched = False url_args = self.args if not url_args: self.RaiseWrongNumberOfArgumentsException() for url_str in url_args: # Throws a CommandException if the argument is not a bucket. bucket_iter = self.GetBucketUrlIterFromArg(url_str) for bucket_listing_ref in bucket_iter: some_matched = True subcommand_func(bucket_listing_ref, subcommand_args) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the ubla command.""" if self.gsutil_api.GetApiSelector(provider='gs') != ApiSelector.JSON: raise CommandException('\n'.join( textwrap.wrap( 'The "%s" command can only be used with the Cloud Storage JSON API.' % self.command_name))) action_subcommand = self.args[0] self.ParseSubOpts(check_args=True) if action_subcommand == 'get' or action_subcommand == 'set': metrics.LogCommandParams(sub_opts=self.sub_opts) metrics.LogCommandParams(subcommands=[action_subcommand]) self._Ubla() else: raise CommandException('Invalid subcommand "%s", use get\|set instead.' % action_subcommand)	10.05 KB	2025-06-25 08:19
📄 update.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of update command for updating gsutil.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil import signal import stat import sys import tarfile import tempfile import textwrap from six.moves import input import gslib from gslib.command import Command from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.metrics import CheckAndMaybePromptForAnalyticsEnabling from gslib.sig_handling import RegisterSignalHandler from gslib.utils import system_util from gslib.utils.boto_util import GetConfigFilePaths from gslib.utils.boto_util import CERTIFICATE_VALIDATION_ENABLED from gslib.utils.constants import RELEASE_NOTES_URL from gslib.utils.text_util import CompareVersions from gslib.utils.update_util import DisallowUpdateIfDataInGsutilDir from gslib.utils.update_util import LookUpGsutilVersion from gslib.utils.update_util import GsutilPubTarball _SYNOPSIS = """ gsutil update [-f] [-n] [url] """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> NOTE: This command is not available if you're using a gsutil installation from a package manager or the Cloud SDK. When using the Cloud SDK, use ``gcloud components update``. The gsutil update command downloads the latest gsutil release, checks its version, and offers to let you update to it if it differs from the version you're currently running. Once you say "Y" to the prompt of whether to install the update, the gsutil update command locates where the running copy of gsutil is installed, unpacks the new version into an adjacent directory, moves the previous version aside, moves the new version to where the previous version was installed, and removes the moved-aside old version. Because of this, users are cautioned not to store data in the gsutil directory, since that data will be lost when you update gsutil. (Some users change directories into the gsutil directory to run the command. We advise against doing that, for this reason.) Note also that the gsutil update command will refuse to run if it finds user data in the gsutil directory. By default gsutil update will retrieve the new code from %s, but you can optionally specify a URL to use instead. This is primarily used for distributing pre-release versions of the code to a small group of early test users. NOTE: gsutil periodically checks whether a more recent software update is available. By default this check is performed every 30 days; you can change (or disable) this check by editing the software_update_check_period variable in the .boto config file. Note also that gsutil will only check for software updates if stdin, stdout, and stderr are all connected to a TTY, to avoid interfering with cron jobs, streaming transfers, and other cases where gsutil input or output are redirected from/to files or pipes. Software update periodic checks are also disabled by the gsutil -q option (see 'gsutil help options') <B>OPTIONS</B> -f Forces the update command to offer to let you update, even if you have the most current copy already. This can be useful if you have a corrupted local copy. -n Causes update command to run without prompting [Y/n] whether to continue if an update is available. """ % GsutilPubTarball()) class UpdateCommand(Command): """Implementation of gsutil update command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'update', command_name_aliases=['refresh'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=1, supported_sub_args='fn', file_url_ok=True, provider_url_ok=False, urls_start_arg=0, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='update', help_name_aliases=['refresh'], help_type='command_help', help_one_line_summary='Update to the latest gsutil release', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def _ExplainIfSudoNeeded(self, tf, dirs_to_remove, old_cwd): """Explains what to do if sudo needed to update gsutil software. Happens if gsutil was previously installed by a different user (typically if someone originally installed in a shared file system location, using sudo). Args: tf: Opened TarFile. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to if sudo is needed. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. Raises: CommandException: if errors encountered. """ # If running under Windows or Cygwin we don't need (or have) sudo. if system_util.IS_CYGWIN or system_util.IS_WINDOWS: return user_id = os.getuid() if os.stat(gslib.GSUTIL_DIR).st_uid == user_id: return # Won't fail - this command runs after main startup code that insists on # having a config file. config_file_list = GetConfigFilePaths() config_files = ' '.join(config_file_list) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) # Pick current protection of each boto config file for command that restores # protection (rather than fixing at 600) to support use cases like how GCE # installs a service account with an /etc/boto.cfg file protected to 644. chmod_cmds = [] for config_file in config_file_list: mode = oct(stat.S_IMODE((os.stat(config_file)[stat.ST_MODE]))) chmod_cmds.append('\n\tsudo chmod %s %s' % (mode, config_file)) raise CommandException('\n'.join( textwrap.wrap( 'Since it was installed by a different user previously, you will need ' 'to update using the following commands. You will be prompted for your ' 'password, and the install will run as "root". If you\'re unsure what ' 'this means please ask your system administrator for help:')) + ( '\n\tsudo chmod 0644 %s\n\tsudo env BOTO_CONFIG="%s" %s update' '%s') % (config_files, config_files, self.gsutil_path, ' '.join(chmod_cmds)), informational=True) # This list is checked during gsutil update by doing a lowercased # slash-left-stripped check. For example "/Dev" would match the "dev" entry. unsafe_update_dirs = [ 'applications', 'auto', 'bin', 'boot', 'desktop', 'dev', 'documents and settings', 'etc', 'export', 'home', 'kernel', 'lib', 'lib32', 'library', 'lost+found', 'mach_kernel', 'media', 'mnt', 'net', 'null', 'network', 'opt', 'private', 'proc', 'program files', 'python', 'root', 'sbin', 'scripts', 'srv', 'sys', 'system', 'tmp', 'users', 'usr', 'var', 'volumes', 'win', 'win32', 'windows', 'winnt', ] def _EnsureDirsSafeForUpdate(self, dirs): """Raises Exception if any of dirs is known to be unsafe for gsutil update. This provides a fail-safe check to ensure we don't try to overwrite or delete any important directories. (That shouldn't happen given the way we construct tmp dirs, etc., but since the gsutil update cleanup uses shutil.rmtree() it's prudent to add extra checks.) Args: dirs: List of directories to check. Raises: CommandException: If unsafe directory encountered. """ for d in dirs: if not d: d = 'null' if d.lstrip(os.sep).lower() in self.unsafe_update_dirs: raise CommandException('EnsureDirsSafeForUpdate: encountered unsafe ' 'directory (%s); aborting update' % d) def _CleanUpUpdateCommand(self, tf, dirs_to_remove, old_cwd): """Cleans up temp files etc. from running update command. Args: tf: Opened TarFile, or None if none currently open. dirs_to_remove: List of directories to remove. old_cwd: Path to the working directory we should chdir back to. It's possible that we've chdir'd to a temp directory that's been deleted, which can cause odd behavior (e.g. OSErrors when opening the metrics subprocess). If this is not truthy, we won't attempt to chdir back to this value. """ if tf: tf.close() self._EnsureDirsSafeForUpdate(dirs_to_remove) for directory in dirs_to_remove: try: shutil.rmtree(directory) except OSError: # Ignore errors while attempting to remove old dirs under Windows. They # happen because of Windows exclusive file locking, and the update # actually succeeds but just leaves the old versions around in the # user's temp dir. if not system_util.IS_WINDOWS: raise if old_cwd: try: os.chdir(old_cwd) except OSError: pass def RunCommand(self): """Command entry point for the update command.""" if gslib.IS_PACKAGE_INSTALL: raise CommandException( 'The update command is only available for gsutil installed from a ' 'tarball. If you installed gsutil via another method, use the same ' 'method to update it.') if system_util.InvokedViaCloudSdk(): raise CommandException( 'The update command is disabled for Cloud SDK installs. Please run ' '"gcloud components update" to update it. Note: the Cloud SDK ' 'incorporates updates to the underlying tools approximately every 2 ' 'weeks, so if you are attempting to update to a recently created ' 'release / pre-release of gsutil it may not yet be available via ' 'the Cloud SDK.') https_validate_certificates = CERTIFICATE_VALIDATION_ENABLED if not https_validate_certificates: raise CommandException( 'Your boto configuration has https_validate_certificates = False.\n' 'The update command cannot be run this way, for security reasons.') DisallowUpdateIfDataInGsutilDir() force_update = False no_prompt = False if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-f': force_update = True if o == '-n': no_prompt = True dirs_to_remove = [] tmp_dir = tempfile.mkdtemp() dirs_to_remove.append(tmp_dir) old_cwd = os.getcwd() os.chdir(tmp_dir) if not no_prompt: self.logger.info('Checking for software update...') if self.args: update_from_url_str = self.args[0] if not update_from_url_str.endswith('.tar.gz'): raise CommandException( 'The update command only works with tar.gz files.') for i, result in enumerate(self.WildcardIterator(update_from_url_str)): if i > 0: raise CommandException( 'Invalid update URL. Must name a single .tar.gz file.') storage_url = result.storage_url if storage_url.IsFileUrl() and not storage_url.IsDirectory(): if not force_update: raise CommandException( ('"update" command does not support "file://" URLs without the ' '-f option.')) elif not (storage_url.IsCloudUrl() and storage_url.IsObject()): raise CommandException( 'Invalid update object URL. Must name a single .tar.gz file.') else: update_from_url_str = GsutilPubTarball() # Try to retrieve version info from tarball metadata; failing that; download # the tarball and extract the VERSION file. The version lookup will fail # when running the update system test, because it retrieves the tarball from # a temp file rather than a cloud URL (files lack the version metadata). tarball_version = LookUpGsutilVersion(self.gsutil_api, update_from_url_str) if tarball_version: tf = None else: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) tf.extractall() with open(os.path.join('gsutil', 'VERSION'), 'r') as ver_file: tarball_version = ver_file.read().strip() if not force_update and gslib.VERSION == tarball_version: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) if self.args: raise CommandException('You already have %s installed.' % update_from_url_str, informational=True) else: raise CommandException( 'You already have the latest gsutil release ' 'installed.', informational=True) if not no_prompt: CheckAndMaybePromptForAnalyticsEnabling() (_, major) = CompareVersions(tarball_version, gslib.VERSION) if major: print(('\n'.join( textwrap.wrap( 'This command will update to the "%s" version of gsutil at %s. ' 'NOTE: This a major new version, so it is strongly recommended ' 'that you review the release note details at %s before updating to ' 'this version, especially if you use gsutil in scripts.' % (tarball_version, gslib.GSUTIL_DIR, RELEASE_NOTES_URL))))) else: print(('This command will update to the "%s" version of\ngsutil at %s' % (tarball_version, gslib.GSUTIL_DIR))) self._ExplainIfSudoNeeded(tf, dirs_to_remove, old_cwd) if no_prompt: answer = 'y' else: answer = input('Proceed? [y/N] ') if not answer or answer.lower()[0] != 'y': self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Not running update.', informational=True) if not tf: tf = self._FetchAndOpenGsutilTarball(update_from_url_str) # Ignore keyboard interrupts during the update to reduce the chance someone # hitting ^C leaves gsutil in a broken state. RegisterSignalHandler(signal.SIGINT, signal.SIG_IGN) # gslib.GSUTIL_DIR lists the path where the code should end up (like # /usr/local/gsutil), which is one level down from the relative path in the # tarball (since the latter creates files in ./gsutil). So, we need to # extract at the parent directory level. gsutil_bin_parent_dir = os.path.normpath( os.path.join(gslib.GSUTIL_DIR, '..')) # Extract tarball to a temporary directory in a sibling to GSUTIL_DIR. old_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) new_dir = tempfile.mkdtemp(dir=gsutil_bin_parent_dir) dirs_to_remove.append(old_dir) dirs_to_remove.append(new_dir) self._EnsureDirsSafeForUpdate(dirs_to_remove) try: tf.extractall(path=new_dir) except Exception as e: self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) raise CommandException('Update failed: %s.' % e) # For enterprise mode (shared/central) installation, users with # different user/group than the installation user/group must be # able to run gsutil so we need to do some permissions adjustments # here. Since enterprise mode is not not supported for Windows # users, we can skip this step when running on Windows, which # avoids the problem that Windows has no find or xargs command. if not system_util.IS_WINDOWS: # Make all files and dirs in updated area owner-RW and world-R, and make # all directories owner-RWX and world-RX. for dirname, subdirs, filenames in os.walk(new_dir): for filename in filenames: fd = os.open(os.path.join(dirname, filename), os.O_RDONLY) os.fchmod(fd, stat.S_IWRITE \| stat.S_IRUSR \| stat.S_IRGRP \| stat.S_IROTH) os.close(fd) for subdir in subdirs: fd = os.open(os.path.join(dirname, subdir), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU \| stat.S_IXGRP \| stat.S_IXOTH \| stat.S_IRGRP \| stat.S_IROTH) os.close(fd) # Make main gsutil script owner-RWX and world-RX. fd = os.open(os.path.join(new_dir, 'gsutil', 'gsutil'), os.O_RDONLY) os.fchmod( fd, stat.S_IRWXU \| stat.S_IRGRP \| stat.S_IXGRP \| stat.S_IROTH \| stat.S_IXOTH) os.close(fd) # Move old installation aside and new into place. os.rename(gslib.GSUTIL_DIR, os.path.join(old_dir, 'old')) os.rename(os.path.join(new_dir, 'gsutil'), gslib.GSUTIL_DIR) self._CleanUpUpdateCommand(tf, dirs_to_remove, old_cwd) RegisterSignalHandler(signal.SIGINT, signal.SIG_DFL) self.logger.info('Update complete.') return 0 def _FetchAndOpenGsutilTarball(self, update_from_url_str): self.command_runner.RunNamedCommand( 'cp', [update_from_url_str, 'file://gsutil.tar.gz'], self.headers, self.debug, skip_update_check=True, ) # Note: tf is closed in _CleanUpUpdateCommand. tf = tarfile.open('gsutil.tar.gz') tf.errorlevel = 1 # So fatal tarball unpack errors raise exceptions. return tf	17.73 KB	2025-06-25 08:19
📄 version.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of gsutil version command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import platform import re import sys import six import boto import crcmod import gslib from gslib.command import Command from gslib.utils import system_util from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import UTF8 from gslib.utils.hashing_helper import GetMd5 from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit _SYNOPSIS = """ gsutil version """ _DETAILED_HELP_TEXT = (""" <B>SYNOPSIS</B> """ + _SYNOPSIS + """ <B>DESCRIPTION</B> Prints information about the version of gsutil. <B>OPTIONS</B> -l Prints additional information, such as the version of Python being used, the version of the Boto library, a checksum of the code, the path to gsutil, and the path to gsutil's configuration file. """) class VersionCommand(Command): """Implementation of gsutil version command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'version', command_name_aliases=['ver'], usage_synopsis=_SYNOPSIS, min_args=0, max_args=0, supported_sub_args='l', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='version', help_name_aliases=['ver'], help_type='command_help', help_one_line_summary='Print version info about gsutil', help_text=_DETAILED_HELP_TEXT, subcommand_help_text={}, ) def RunCommand(self): """Command entry point for the version command.""" long_form = False if self.sub_opts: for o, _ in self.sub_opts: if o == '-l': long_form = True config_paths = ', '.join(GetFriendlyConfigFilePaths()) shipped_checksum = gslib.CHECKSUM try: cur_checksum = self._ComputeCodeChecksum() except IOError: cur_checksum = 'MISSING FILES' if shipped_checksum == cur_checksum: checksum_ok_str = 'OK' else: checksum_ok_str = '!= %s' % shipped_checksum sys.stdout.write('gsutil version: %s\n' % gslib.VERSION) if long_form: long_form_output = ( 'checksum: {checksum} ({checksum_ok})\n' 'boto version: {boto_version}\n' 'python version: {python_version}\n' 'OS: {os_version}\n' 'multiprocessing available: {multiprocessing_available}\n' 'using cloud sdk: {cloud_sdk}\n' 'pass cloud sdk credentials to gsutil: {cloud_sdk_credentials}\n' 'config path(s): {config_paths}\n' 'gsutil path: {gsutil_path}\n' 'compiled crcmod: {compiled_crcmod}\n' 'installed via package manager: {is_package_install}\n' 'editable install: {is_editable_install}\n' 'shim enabled: {is_shim_enabled}\n') sys.stdout.write( long_form_output.format( checksum=cur_checksum, checksum_ok=checksum_ok_str, boto_version=boto.__version__, python_version=sys.version.replace('\n', ''), os_version='%s %s' % (platform.system(), platform.release()), multiprocessing_available=( CheckMultiprocessingAvailableAndInit().is_available), cloud_sdk=system_util.InvokedViaCloudSdk(), cloud_sdk_credentials=system_util.CloudSdkCredPassingEnabled(), config_paths=config_paths, gsutil_path=(GetCloudSdkGsutilWrapperScriptPath() or gslib.GSUTIL_PATH), compiled_crcmod=UsingCrcmodExtension(), is_package_install=gslib.IS_PACKAGE_INSTALL, is_editable_install=gslib.IS_EDITABLE_INSTALL, is_shim_enabled=boto.config.getbool('GSUtil', 'use_gcloud_storage', False))) return 0 def _ComputeCodeChecksum(self): """Computes a checksum of gsutil code. This checksum can be used to determine if users locally modified gsutil when requesting support. (It's fine for users to make local mods, but when users ask for support we ask them to run a stock version of gsutil so we can reduce possible variables.) Returns: MD5 checksum of gsutil code. """ if gslib.IS_PACKAGE_INSTALL: return 'PACKAGED_GSUTIL_INSTALLS_DO_NOT_HAVE_CHECKSUMS' m = GetMd5() # Checksum gsutil and all .py files under gslib directory. files_to_checksum = [gslib.GSUTIL_PATH] for root, _, files in os.walk(gslib.GSLIB_DIR): for filepath in files: if filepath.endswith('.py'): files_to_checksum.append(os.path.join(root, filepath)) # Sort to ensure consistent checksum build, no matter how os.walk # orders the list. for filepath in sorted(files_to_checksum): if six.PY2: f = open(filepath, 'rb') content = f.read() content = re.sub(r'(\r\n\|\r\|\n)', b'\n', content) m.update(content) f.close() else: f = open(filepath, 'r', encoding=UTF8) content = f.read() content = re.sub(r'(\r\n\|\r\|\n)', '\n', content) m.update(content.encode(UTF8)) f.close() return m.hexdigest() def GetCloudSdkGsutilWrapperScriptPath(): """If gsutil was invoked via the Cloud SDK, find its gsutil wrapper script. Returns: (str) The path to the Cloud SDK's gsutil wrapper script, or an empty string if gsutil was not invoked via the Cloud SDK or the wrapper script could not be found at its expected path. """ # If running via the Cloud SDK, the "real" gsutil script was probably invoked # indirectly through gcloud's gsutil wrapper script (the former is at # <sdk-root>/platform/gsutil/gsutil, while the latter is located at # <sdk-root>/bin/gsutil). We should print the wrapper script's path if it # exists and we were invoked via the Cloud SDK. gsutil_path = gslib.GSUTIL_PATH if system_util.InvokedViaCloudSdk(): platform_path_suffix = os.path.join('platform', 'gsutil', 'gsutil') if gsutil_path.endswith(platform_path_suffix): bin_path = os.path.join( gsutil_path[0:gsutil_path.rfind(platform_path_suffix)], 'bin', 'gsutil', ) if os.path.exists(bin_path): return bin_path return ''	7.12 KB	2025-06-25 08:19
📄 versioning.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of versioning configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageMap from gslib.utils import shim_util _SET_SYNOPSIS = """ gsutil versioning set (on\|off) gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil versioning get gs://<bucket_name>... """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The "set" sub-command requires an additional sub-command, either "on" or "off", which, respectively, will enable or disable versioning for the specified bucket(s). """ _GET_DESCRIPTION = """ <B>GET</B> The "get" sub-command gets the versioning configuration for a bucket and displays whether or not it is enabled. """ _DESCRIPTION = """ The `Versioning Configuration <https://cloud.google.com/storage/docs/object-versioning>`_ feature enables you to configure a Google Cloud Storage bucket to keep old versions of objects. The gsutil versioning command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _GCLOUD_FORMAT_STRING = ('--format=value[separator=""](' + 'name.sub("' + shim_util.get_format_flag_caret() + '", "gs://").sub("$", ": "),' + 'versioning.enabled' + '.yesno("Enabled", "Suspended"))') _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class VersioningCommand(Command): """Implementation of gsutil versioning command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'versioning', command_name_aliases=['setversioning', 'getversioning'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=2, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [ CommandArgument('mode', choices=['on', 'off']), CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument() ], 'get': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='versioning', help_name_aliases=[ 'getversioning', 'setversioning', ], help_type='command_help', help_one_line_summary=( 'Enable or suspend versioning for one or more buckets'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'list', '--raw', _GCLOUD_FORMAT_STRING ], flag_map={}, supports_output_translation=True, ), 'set': GcloudStorageMap( gcloud_command={ 'on': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--versioning', ], flag_map={}, ), 'off': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', '--no-versioning', ], flag_map={}, ), }, flag_map={}, ) }, flag_map={}, ) def _CalculateUrlsStartArg(self): if not self.args: self.RaiseWrongNumberOfArgumentsException() if self.args[0].lower() == 'set': return 2 else: return 1 def _SetVersioning(self): """Gets versioning configuration for a bucket.""" versioning_arg = self.args[0].lower() if versioning_arg not in ('on', 'off'): raise CommandException('Argument to "%s set" must be either <on\|off>' % (self.command_name)) url_args = self.args[1:] if not url_args: self.RaiseWrongNumberOfArgumentsException() # Iterate over URLs, expanding wildcards and set the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True bucket_metadata = apitools_messages.Bucket( versioning=apitools_messages.Bucket.VersioningValue()) if versioning_arg == 'on': self.logger.info('Enabling versioning for %s...', url) bucket_metadata.versioning.enabled = True else: self.logger.info('Suspending versioning for %s...', url) bucket_metadata.versioning.enabled = False self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def _GetVersioning(self): """Gets versioning configuration for one or more buckets.""" url_args = self.args # Iterate over URLs, expanding wildcards and getting the versioning # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['versioning']) for blr in bucket_iter: some_matched = True if blr.root_object.versioning and blr.root_object.versioning.enabled: print('%s: Enabled' % blr.url_string.rstrip('/')) else: print('%s: Suspended' % blr.url_string.rstrip('/')) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) def RunCommand(self): """Command entry point for the versioning command.""" action_subcommand = self.args.pop(0) if action_subcommand == 'get': func = self._GetVersioning metrics.LogCommandParams(subcommands=[action_subcommand]) elif action_subcommand == 'set': func = self._SetVersioning versioning_arg = self.args[0].lower() if versioning_arg in ('on', 'off'): metrics.LogCommandParams( subcommands=[action_subcommand, versioning_arg]) else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help %s".') % (action_subcommand, self.command_name, self.command_name)) func() return 0	8.34 KB	2025-06-25 08:19
📄 web.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of website configuration command for buckets.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys from apitools.base.py import encoding from gslib import metrics from gslib.command import Command from gslib.command_argument import CommandArgument from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.help_provider import CreateHelpText from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import NO_MAX from gslib.utils.shim_util import GcloudStorageFlag from gslib.utils.shim_util import GcloudStorageMap _SET_SYNOPSIS = """ gsutil web set [-m <main_page_suffix>] [-e <error_page>] gs://<bucket_name>... """ _GET_SYNOPSIS = """ gsutil web get gs://<bucket_name> """ _SYNOPSIS = _SET_SYNOPSIS + _GET_SYNOPSIS.lstrip('\n') _SET_DESCRIPTION = """ <B>SET</B> The ``gsutil web set`` command allows you to configure or disable the website configuration on your buckets. The ``set`` sub-command has the following options (omit both options to unset the configuration): <B>SET OPTIONS</B> -m <index.html> Specifies the object name to serve when a bucket listing is requested via a custom domain. -e <404.html> Specifies the error page to serve when a request is made for a non-existent object via a custom domain """ _GET_DESCRIPTION = """ <B>GET</B> The ``gsutil web get`` command retrieves the web semantics configuration for a bucket and displays a JSON representation of the configuration. In Google Cloud Storage, this would look like the following: { "notFoundPage": "404.html", "mainPageSuffix": "index.html" } """ _DESCRIPTION = """ Cloud Storage allows you to configure a bucket to behave like a static website when the bucket is accessed through a `custom domain <https://cloud.google.com/storage/docs/request-endpoints#custom-domains>`_. For example, if you set a ``main_page_suffix``, a subsequent GET bucket request through a custom domain serves the specified "main" page instead of performing the usual bucket listing. Similarly, if you set an ``error_page``, a subsequent GET object request through a custom domain for a non-existent object serves the specified error page instead of the standard Cloud Storage error. Requests to a bucket through other `endpoints <https://cloud.google.com/storage/docs/request-endpoints>`_ are unaffected by the bucket's website configuration. See `Static website examples and tips <https://cloud.google.com/storage/docs/static-website>`_ for additional examples and information. Notes: 1. Because the main page is only served when a bucket listing request is made via a custom domain endpoint, you can continue to use ``gsutil ls`` as you normally would for bucket listing. 2. The main_page_suffix applies to each subdirectory of the bucket. For example, with the main_page_suffix configured to be index.html, a GET request for ``http://www.example.com`` retrieves ``http://www.example.com/index.html`` but shows ``http://www.example.com`` as the URL to the requester, and a GET request for ``http://www.example.com/photos`` retrieves ``http://www.example.com/photos/index.html`` but shows ``http://www.example.com/photos`` as the URL to the requester. 3. There is just one 404.html page. For example, a GET request for ``http://www.example.com/photos/missing`` retrieves ``http://www.example.com/404.html``, not ``http://www.example.com/photos/404.html``. The web command has two sub-commands: """ + _SET_DESCRIPTION + _GET_DESCRIPTION _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION) _get_help_text = CreateHelpText(_GET_SYNOPSIS, _GET_DESCRIPTION) _set_help_text = CreateHelpText(_SET_SYNOPSIS, _SET_DESCRIPTION) class WebCommand(Command): """Implementation of gsutil web command.""" # Command specification. See base class for documentation. command_spec = Command.CreateCommandSpec( 'web', command_name_aliases=['setwebcfg', 'getwebcfg'], usage_synopsis=_SYNOPSIS, min_args=2, max_args=NO_MAX, supported_sub_args='m:e:', file_url_ok=False, provider_url_ok=False, urls_start_arg=1, gs_api_support=[ApiSelector.XML, ApiSelector.JSON], gs_default_api=ApiSelector.JSON, argparse_arguments={ 'set': [CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()], 'get': [CommandArgument.MakeNCloudBucketURLsArgument(1)], }, ) # Help specification. See help_provider.py for documentation. help_spec = Command.HelpSpec( help_name='web', help_name_aliases=['getwebcfg', 'setwebcfg'], help_type='command_help', help_one_line_summary=('Set a website configuration for a bucket'), help_text=_DETAILED_HELP_TEXT, subcommand_help_text={ 'get': _get_help_text, 'set': _set_help_text, }, ) gcloud_storage_map = GcloudStorageMap( gcloud_command={ 'get': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'describe', '--format=gsutiljson[key=website_config,empty=\' has ' 'no website configuration.\',empty_prefix_key=' 'storage_url]' ], flag_map={}, supports_output_translation=True, ), # "set" subcommand handled in get_gcloud_storage_args. }, flag_map={}, ) def get_gcloud_storage_args(self): if self.args[0] == 'set': set_command_map = GcloudStorageMap( gcloud_command={ 'set': GcloudStorageMap( gcloud_command=[ 'storage', 'buckets', 'update', ], flag_map={ '-e': GcloudStorageFlag('--web-error-page'), '-m': GcloudStorageFlag('--web-main-page-suffix'), }) }, flag_map={}, ) if not ('-e' in self.args or '-m' in self.args): set_command_map.gcloud_command['set'].gcloud_command += [ '--clear-web-error-page', '--clear-web-main-page-suffix' ] gcloud_storage_map = set_command_map else: gcloud_storage_map = WebCommand.gcloud_storage_map return super().get_gcloud_storage_args(gcloud_storage_map) def _GetWeb(self): """Gets website configuration for a bucket.""" bucket_url, bucket_metadata = self.GetSingleBucketUrlFromArg( self.args[0], bucket_fields=['website']) if bucket_url.scheme == 's3': sys.stdout.write( self.gsutil_api.XmlPassThroughGetWebsite(bucket_url, provider=bucket_url.scheme)) else: if bucket_metadata.website and (bucket_metadata.website.mainPageSuffix or bucket_metadata.website.notFoundPage): sys.stdout.write( str(encoding.MessageToJson(bucket_metadata.website)) + '\n') else: sys.stdout.write('%s has no website configuration.\n' % bucket_url) return 0 def _SetWeb(self): """Sets website configuration for a bucket.""" main_page_suffix = None error_page = None if self.sub_opts: for o, a in self.sub_opts: if o == '-m': main_page_suffix = a elif o == '-e': error_page = a url_args = self.args website = apitools_messages.Bucket.WebsiteValue( mainPageSuffix=main_page_suffix, notFoundPage=error_page) # Iterate over URLs, expanding wildcards and setting the website # configuration on each. some_matched = False for url_str in url_args: bucket_iter = self.GetBucketUrlIterFromArg(url_str, bucket_fields=['id']) for blr in bucket_iter: url = blr.storage_url some_matched = True self.logger.info('Setting website configuration on %s...', blr) bucket_metadata = apitools_messages.Bucket(website=website) self.gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) if not some_matched: raise CommandException(NO_URLS_MATCHED_TARGET % list(url_args)) return 0 def RunCommand(self): """Command entry point for the web command.""" action_subcommand = self.args.pop(0) self.ParseSubOpts(check_args=True) if action_subcommand == 'get': func = self._GetWeb elif action_subcommand == 'set': func = self._SetWeb else: raise CommandException( ('Invalid subcommand "%s" for the %s command.\n' 'See "gsutil help web".') % (action_subcommand, self.command_name)) # Commands with both suboptions and subcommands need to reparse for # suboptions, so we log again. metrics.LogCommandParams(subcommands=[action_subcommand], sub_opts=self.sub_opts) return func()	9.87 KB	2025-06-25 08:19

Edit File:

Terminal