Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-06-25 08:26
📄 __init__.py	0 B	2025-06-25 08:19
📄 acl_helper.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper objects for changing and deleting ACLs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import re from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages class ChangeType(object): USER = 'User' GROUP = 'Group' PROJECT = 'Project' class AclChange(object): """Represents a logical change to an access control list.""" public_scopes = ['AllAuthenticatedUsers', 'AllUsers'] id_scopes = ['UserById', 'GroupById'] email_scopes = ['UserByEmail', 'GroupByEmail'] domain_scopes = ['GroupByDomain'] project_scopes = ['Project'] scope_types = (public_scopes + id_scopes + email_scopes + domain_scopes + project_scopes) public_entity_all_users = 'allUsers' public_entity_all_auth_users = 'allAuthenticatedUsers' public_entity_types = (public_entity_all_users, public_entity_all_auth_users) project_entity_prefixes = ('project-editors-', 'project-owners-', 'project-viewers-') group_entity_prefix = 'group-' user_entity_prefix = 'user-' domain_entity_prefix = 'domain-' project_entity_prefix = 'project-' permission_shorthand_mapping = { 'R': 'READER', 'W': 'WRITER', 'FC': 'OWNER', 'O': 'OWNER', 'READ': 'READER', 'WRITE': 'WRITER', 'FULL_CONTROL': 'OWNER', } def __init__(self, acl_change_descriptor, scope_type): """Creates an AclChange object. Args: acl_change_descriptor: An acl change as described in the "ch" section of the "acl" command's help. scope_type: Either ChangeType.USER or ChangeType.GROUP or ChangeType.PROJECT, specifying the extent of the scope. """ self.identifier = '' self.raw_descriptor = acl_change_descriptor self._Parse(acl_change_descriptor, scope_type) self._Validate() def __str__(self): return 'AclChange<{0}\|{1}\|{2}>'.format(self.scope_type, self.perm, self.identifier) def _Parse(self, change_descriptor, scope_type): """Parses an ACL Change descriptor.""" def _ClassifyScopeIdentifier(text): re_map = { 'AllAuthenticatedUsers': r'^(AllAuthenticatedUsers\|AllAuth)$', 'AllUsers': '^(AllUsers\|All)$', 'Email': r'^.+@.+\..+$', 'Id': r'^[0-9A-Fa-f]{64}$', 'Domain': r'^[^@]+\.[^@]+$', 'Project': r'(owners\|editors\|viewers)\-.+$', } for type_string, regex in re_map.items(): if re.match(regex, text, re.IGNORECASE): return type_string if change_descriptor.count(':') != 1: raise CommandException( '{0} is an invalid change description.'.format(change_descriptor)) scope_string, perm_token = change_descriptor.split(':') perm_token = perm_token.upper() if perm_token in self.permission_shorthand_mapping: self.perm = self.permission_shorthand_mapping[perm_token] else: self.perm = perm_token scope_class = _ClassifyScopeIdentifier(scope_string) if scope_class == 'Domain': # This may produce an invalid UserByDomain scope, # which is good because then validate can complain. self.scope_type = '{0}ByDomain'.format(scope_type) self.identifier = scope_string elif scope_class in ('Email', 'Id'): self.scope_type = '{0}By{1}'.format(scope_type, scope_class) self.identifier = scope_string elif scope_class == 'AllAuthenticatedUsers': self.scope_type = 'AllAuthenticatedUsers' elif scope_class == 'AllUsers': self.scope_type = 'AllUsers' elif scope_class == 'Project': self.scope_type = 'Project' self.identifier = scope_string else: # This is just a fallback, so we set it to something # and the validate step has something to go on. self.scope_type = scope_string def _Validate(self): """Validates a parsed AclChange object.""" def _ThrowError(msg): raise CommandException('{0} is not a valid ACL change\n{1}'.format( self.raw_descriptor, msg)) if self.scope_type not in self.scope_types: _ThrowError('{0} is not a valid scope type'.format(self.scope_type)) if self.scope_type in self.public_scopes and self.identifier: _ThrowError('{0} requires no arguments'.format(self.scope_type)) if self.scope_type in self.id_scopes and not self.identifier: _ThrowError('{0} requires an id'.format(self.scope_type)) if self.scope_type in self.email_scopes and not self.identifier: _ThrowError('{0} requires an email address'.format(self.scope_type)) if self.scope_type in self.domain_scopes and not self.identifier: _ThrowError('{0} requires domain'.format(self.scope_type)) if self.perm not in self.permission_shorthand_mapping.values(): perms = ', '.join(set(self.permission_shorthand_mapping.values())) _ThrowError('Allowed permissions are {0}'.format(perms)) def _YieldMatchingEntries(self, current_acl): """Generator that yields entries that match the change descriptor. Args: current_acl: A list of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. """ for entry in current_acl: if (self.scope_type in ('UserById', 'GroupById') and entry.entityId and self.identifier == entry.entityId): yield entry elif (self.scope_type in ('UserByEmail', 'GroupByEmail') and entry.email and self.identifier == entry.email): yield entry elif (self.scope_type == 'GroupByDomain' and entry.domain and self.identifier == entry.domain): yield entry elif (self.scope_type == 'Project' and entry.projectTeam and self.identifier == '%s-%s' % (entry.projectTeam.team, entry.projectTeam.projectNumber)): yield entry elif (self.scope_type == 'AllUsers' and entry.entity.lower() == self.public_entity_all_users.lower()): yield entry elif (self.scope_type == 'AllAuthenticatedUsers' and entry.entity.lower() == self.public_entity_all_auth_users.lower()): yield entry def GetEntity(self): """Gets an appropriate entity string for an ACL grant.""" if self.scope_type in ('UserById', 'UserByEmail'): return self.user_entity_prefix + self.identifier elif self.scope_type in ('GroupById', 'GroupByEmail'): return self.group_entity_prefix + self.identifier elif self.scope_type == 'Project': return self.project_entity_prefix + self.identifier elif self.scope_type == 'GroupByDomain': return self.domain_entity_prefix + self.identifier elif self.scope_type == 'AllAuthenticatedUsers': return self.public_entity_all_auth_users elif self.scope_type == 'AllUsers': return self.public_entity_all_users else: raise CommandException('Add entry to ACL got unexpected scope type %s.' % self.scope_type) def _AddEntry(self, current_acl, entry_class): """Adds an entry to current_acl.""" entity = self.GetEntity() entry = entry_class(role=self.perm, entity=entity) if self.scope_type in ('UserById', 'GroupById', 'Project'): entry.entityId = self.identifier elif self.scope_type in ('UserByEmail', 'GroupByEmail'): entry.email = self.identifier elif self.scope_type == 'GroupByDomain': entry.domain = self.identifier current_acl.append(entry) def _GetEntriesClass(self, current_acl): # Entries will share the same class, so just return the first one. for acl_entry in current_acl: return acl_entry.__class__ # It's possible that a default object ACL is empty, so if we have # an empty list, assume it is an object ACL. return apitools_messages.ObjectAccessControl().__class__ def Execute(self, storage_url, current_acl, command_name, logger): """Executes the described change on an ACL. Args: storage_url: StorageUrl representing the object to change. current_acl: A list of ObjectAccessControls or BucketAccessControls to permute. command_name: String name of comamnd being run (e.g., 'acl'). logger: An instance of logging.Logger. Returns: The number of changes that were made. """ logger.debug('Executing %s %s on %s', command_name, self.raw_descriptor, storage_url) if self.perm == 'WRITER': if command_name == 'acl' and storage_url.IsObject(): logger.warning('Skipping %s on %s, as WRITER does not apply to objects', self.raw_descriptor, storage_url) return 0 elif command_name == 'defacl': raise CommandException('WRITER cannot be set as a default object ACL ' 'because WRITER does not apply to objects') entry_class = self._GetEntriesClass(current_acl) matching_entries = list(self._YieldMatchingEntries(current_acl)) change_count = 0 if matching_entries: for entry in matching_entries: if entry.role != self.perm: entry.role = self.perm change_count += 1 else: self._AddEntry(current_acl, entry_class) change_count = 1 logger.debug('New Acl:\n%s', str(current_acl)) return change_count class AclDel(object): """Represents a logical change from an access control list.""" scope_regexes = { r'All(Users)?$': 'AllUsers', r'AllAuth(enticatedUsers)?$': 'AllAuthenticatedUsers', } def __init__(self, identifier): self.raw_descriptor = '-d {0}'.format(identifier) self.identifier = identifier for regex, scope in self.scope_regexes.items(): if re.match(regex, self.identifier, re.IGNORECASE): self.identifier = scope self.scope_type = 'Any' self.perm = 'NONE' def _YieldMatchingEntries(self, current_acl): """Generator that yields entries that match the change descriptor. Args: current_acl: An instance of apitools_messages.BucketAccessControls or ObjectAccessControls which will be searched for matching entries. Yields: An apitools_messages.BucketAccessControl or ObjectAccessControl. """ for entry in current_acl: if entry.entityId and self.identifier.lower() == entry.entityId.lower(): yield entry elif entry.email and self.identifier.lower() == entry.email.lower(): yield entry elif entry.domain and self.identifier.lower() == entry.domain.lower(): yield entry elif (entry.projectTeam and self.identifier.lower() == '%s-%s'.lower() % (entry.projectTeam.team, entry.projectTeam.projectNumber)): yield entry elif entry.entity.lower() == 'allusers' and self.identifier == 'AllUsers': yield entry elif (entry.entity.lower() == 'allauthenticatedusers' and self.identifier == 'AllAuthenticatedUsers'): yield entry def Execute(self, storage_url, current_acl, command_name, logger): logger.debug('Executing %s %s on %s', command_name, self.raw_descriptor, storage_url) matching_entries = list(self._YieldMatchingEntries(current_acl)) for entry in matching_entries: current_acl.remove(entry) logger.debug('New Acl:\n%s', str(current_acl)) return len(matching_entries) def translate_sub_opts_for_shim(sub_opts): translated_sub_opts = [] scope_type_from_flag = { '-g': ChangeType.GROUP, '-p': ChangeType.PROJECT, '-u': ChangeType.USER, } for (flag, value) in sub_opts: if flag in scope_type_from_flag: change = AclChange(value, scope_type=scope_type_from_flag[flag]) new_value = 'entity={},role={}'.format(change.GetEntity(), change.perm) translated_sub_opts.append((flag, new_value)) elif flag == '-d': change = AclDel(value) translated_sub_opts.append(('-d', change.identifier)) else: translated_sub_opts.append((flag, value)) return translated_sub_opts	12.66 KB	2025-06-25 08:19
📄 arg_helper.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper for parsing command arguments and options.""" import getopt import sys from gslib.exception import CommandException def GetArgumentsAndOptions(): """Gets the list of arguments and options from the command input. Returns: The return value consists of two elements: the first is a list of (option, value) pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of the first argument). """ try: return getopt.getopt(sys.argv[1:], 'dDvo:?h:i:u:mq', [ 'debug', 'detailedDebug', 'version', 'option', 'help', 'header', 'impersonate-service-account=', 'multithreaded', 'quiet', 'testexceptiontraces', 'trace-token=', 'perf-trace-token=' ]) except getopt.GetoptError as e: raise CommandException(e.msg)	1.42 KB	2025-06-25 08:19
📄 boto_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods that require importing boto. This module also imports httplib2 (as it is Boto's http transport and closely tied to some of Boto's core functionality) and oauth2client. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import pkgutil import tempfile import textwrap import six import boto from boto import config import boto.auth from boto.exception import NoAuthHandlerFound from boto.gs.connection import GSConnection from boto.provider import Provider from boto.pyami.config import BotoConfigLocations import gslib from gslib import context_config from gslib.exception import CommandException from gslib.utils import system_util from gslib.utils.constants import DEFAULT_GCS_JSON_API_VERSION from gslib.utils.constants import DEFAULT_GSUTIL_STATE_DIR from gslib.utils.constants import SSL_TIMEOUT_SEC from gslib.utils.constants import UTF8 from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import ONE_MIB import httplib2 from oauth2client.client import HAS_CRYPTO if six.PY3: long = int # Globals in this module are set according to values in the boto config. BOTO_IS_SECURE = config.get('Boto', 'is_secure', True) CERTIFICATE_VALIDATION_ENABLED = config.get('Boto', 'https_validate_certificates', True) configured_certs_file = None # Single certs file for use across all processes. temp_certs_file = None # Temporary certs file for cleanup upon exit. def ConfigureCertsFile(): """Configures and returns the CA Certificates file. If one is already configured, use it. Otherwise, use the cert roots distributed with gsutil. Returns: string filename of the certs file to use. """ certs_file = boto.config.get('Boto', 'ca_certificates_file', None) # The 'system' keyword indicates to use the system installed certs. Some # Linux distributions patch the stack such that the Python SSL # infrastructure picks up the system installed certs by default, thus no # action necessary on our part if certs_file == 'system': return None if not certs_file: global configured_certs_file, temp_certs_file if not configured_certs_file: configured_certs_file = os.path.abspath( os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt')) if not os.path.exists(configured_certs_file): # If the file is not present on disk, this means the gslib module # doesn't actually exist on disk anywhere. This can happen if it's # being imported from a zip file. Unfortunately, we have to copy the # certs file to a local temp file on disk because the underlying SSL # socket requires it to be a filesystem path. certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt') if not certs_data: raise CommandException('Certificates file not found. Please ' 'reinstall gsutil from scratch') certs_data = six.ensure_str(certs_data) fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts') f = os.fdopen(fd, 'w') f.write(certs_data) f.close() temp_certs_file = fname configured_certs_file = temp_certs_file certs_file = configured_certs_file return certs_file def ConfigureNoOpAuthIfNeeded(): """Sets up no-op auth handler if no boto credentials are configured.""" if not HasConfiguredCredentials(): if (config.has_option('Credentials', 'gs_service_client_id') and not HAS_CRYPTO): if system_util.InvokedViaCloudSdk(): raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please reactivate your service account via the gcloud auth ' 'command and ensure any gcloud packages necessary for ' 'service accounts are present.'))) else: raise CommandException('\n'.join( textwrap.wrap( 'Your gsutil is configured with an OAuth2 service account, but ' 'you do not have PyOpenSSL or PyCrypto 2.6 or later installed. ' 'Service account authentication requires one of these libraries; ' 'please install either of them to proceed, or configure a ' 'different type of credentials with "gsutil config".'))) else: # With no boto config file the user can still access publicly readable # buckets and objects. from gslib import no_op_auth_plugin # pylint: disable=unused-variable def GetCertsFile(): return configured_certs_file def GetCleanupFiles(): """Returns a list of temp files to delete (if possible) when program exits.""" return [temp_certs_file] if temp_certs_file else [] def GetConfigFilePaths(): """Returns a list of the path(s) to the boto config file(s) to be loaded.""" potential_config_paths = BotoConfigLocations # When Boto's pyami.config.Config class is initialized, it attempts to read # this file, transform it into valid Boto config syntax, and load credentials # from it, but it does not add this file to BotoConfigLocations. if 'AWS_CREDENTIAL_FILE' in os.environ: potential_config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) # Provider-specific config files, like the one below, are checked for and # loaded when Boto's Provider class is initialized. These aren't listed in # BotoConfigLocations, but can still affect what credentials are loaded, so # we display them in our config list to make auth'n debugging easier. aws_cred_file = os.path.join(os.path.expanduser('~'), '.aws', 'credentials') if os.path.isfile(aws_cred_file): potential_config_paths.append(aws_cred_file) # Only return credential files which we have permission to read (and thus can # actually load credentials from). readable_config_paths = [] for path in potential_config_paths: try: with open(path, 'r'): readable_config_paths.append(path) except IOError: pass return readable_config_paths def GetFriendlyConfigFilePaths(): """Like GetConfigFilePaths but returns a not-found message if paths empty.""" readable_config_paths = GetConfigFilePaths() if len(readable_config_paths) == 0: readable_config_paths.append('No config found') return readable_config_paths def GetGsutilStateDir(): """Returns the location of the directory for gsutil state files. Certain operations, such as cross-process credential sharing and resumable transfer tracking, need a known location for state files which are created by gsutil as-needed. This location should only be used for storing data that is required to be in a static location. Returns: Path to directory for gsutil static state files. """ config_file_dir = config.get('GSUtil', 'state_dir', DEFAULT_GSUTIL_STATE_DIR) system_util.CreateDirIfNeeded(config_file_dir) return config_file_dir def GetCredentialStoreFilename(): # As of gsutil 4.29, this changed from 'credstore' to 'credstore2' because # of a change to the underlying credential storage format. return os.path.join(GetGsutilStateDir(), 'credstore2') def GetGceCredentialCacheFilename(): return os.path.join(GetGsutilStateDir(), 'gcecredcache') def GetGcsJsonApiVersion(): return config.get('GSUtil', 'json_api_version', DEFAULT_GCS_JSON_API_VERSION) # Resumable downloads and uploads make one HTTP call per chunk (and must be # in multiples of 256KiB). Overridable for testing. def GetJsonResumableChunkSize(): chunk_size = config.getint('GSUtil', 'json_resumable_chunk_size', long(1024 * 1024 * 100)) if chunk_size == 0: chunk_size = long(1024 * 256) elif chunk_size % long(1024 * 256) != 0: chunk_size += (long(1024 * 256) - (chunk_size % (long(1024 * 256)))) return chunk_size def GetLastCheckedForGsutilUpdateTimestampFile(): return os.path.join(GetGsutilStateDir(), '.last_software_update_check') def GetMaxConcurrentCompressedUploads(): """Gets the max concurrent transport compressed uploads allowed in parallel. Returns: The max number of concurrent transport compressed uploads allowed in parallel without exceeding the max_upload_compression_buffer_size. """ upload_chunk_size = GetJsonResumableChunkSize() # From apitools compression.py. compression_chunk_size = 16 * ONE_MIB total_upload_size = (upload_chunk_size + compression_chunk_size + 17 + 5 * (((compression_chunk_size - 1) / 16383) + 1)) max_concurrent_uploads = (GetMaxUploadCompressionBufferSize() / total_upload_size) if max_concurrent_uploads <= 0: max_concurrent_uploads = 1 return max_concurrent_uploads def GetMaxRetryDelay(): return config.getint('Boto', 'max_retry_delay', 32) def GetMaxUploadCompressionBufferSize(): """Get the max amount of memory compressed transport uploads may buffer.""" return HumanReadableToBytes( config.get('GSUtil', 'max_upload_compression_buffer_size', '2GiB')) def GetNewHttp(http_class=httplib2.Http, kwargs): """Creates and returns a new httplib2.Http instance. Args: http_class: Optional custom Http class to use. kwargs: Arguments to pass to http_class constructor. Returns: An initialized httplib2.Http instance. """ ##Get Proxy configuration from boto file, defaults are None, 0 and False proxy_host = config.get('Boto', 'proxy', None) #needed for if clause below boto_proxy_config = { 'proxy_host': proxy_host, 'proxy_type': config.get('Boto', 'proxy_type', 'http'), 'proxy_port': config.getint('Boto', 'proxy_port'), 'proxy_user': config.get('Boto', 'proxy_user', None), 'proxy_pass': config.get('Boto', 'proxy_pass', None), 'proxy_rdns': config.get('Boto', 'proxy_rdns', True if proxy_host else None) } #Use SetProxyInfo to convert boto config to httplib2.proxyinfo object proxy_info = SetProxyInfo(boto_proxy_config) # Some installers don't package a certs file with httplib2, so use the # one included with gsutil. kwargs['ca_certs'] = GetCertsFile() # Use a non-infinite SSL timeout to avoid hangs during network flakiness. kwargs['timeout'] = SSL_TIMEOUT_SEC http = http_class(proxy_info=proxy_info, *kwargs) http.disable_ssl_certificate_validation = (not config.getbool( 'Boto', 'https_validate_certificates')) global_context_config = context_config.get_context_config() if global_context_config and global_context_config.use_client_certificate: http.add_certificate(key=global_context_config.client_cert_path, cert=global_context_config.client_cert_path, domain='', password=global_context_config.client_cert_password) return http # Retry for 10 minutes with exponential backoff, which corresponds to # the maximum Downtime Period specified in the GCS SLA # (https://cloud.google.com/storage/sla) def GetNumRetries(): return config.getint('Boto', 'num_retries', 23) def GetTabCompletionLogFilename(): return os.path.join(GetGsutilStateDir(), 'tab-completion-logs') def GetTabCompletionCacheFilename(): tab_completion_dir = os.path.join(GetGsutilStateDir(), 'tab-completion') # Limit read permissions on the directory to owner for privacy. system_util.CreateDirIfNeeded(tab_completion_dir, mode=0o700) return os.path.join(tab_completion_dir, 'cache') def HasConfiguredCredentials(): """Determines if boto credential/config file exists.""" has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and config.has_option('Credentials', 'aws_secret_access_key')) has_oauth_creds = (config.has_option('Credentials', 'gs_oauth2_refresh_token')) has_external_creds = (config.has_option('Credentials', 'gs_external_account_file')) has_external_account_authorized_user_creds = (config.has_option( 'Credentials', 'gs_external_account_authorized_user_file')) has_service_account_creds = ( HAS_CRYPTO and config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) if (has_goog_creds or has_amzn_creds or has_oauth_creds or has_service_account_creds or has_external_creds or has_external_account_authorized_user_creds): return True valid_auth_handler = None try: valid_auth_handler = boto.auth.get_auth_handler(GSConnection.DefaultHost, config, Provider('google'), requested_capability=['s3']) # Exclude the no-op auth handler as indicating credentials are configured. # Note we can't use isinstance() here because the no-op module may not be # imported so we can't get a reference to the class type. if 'NoOpAuth' == getattr( getattr(valid_auth_handler, '__class__', None), '__name__', None): # yapf: disable valid_auth_handler = None except NoAuthHandlerFound: pass return valid_auth_handler def JsonResumableChunkSizeDefined(): chunk_size_defined = config.get('GSUtil', 'json_resumable_chunk_size', None) return chunk_size_defined is not None def MonkeyPatchBoto(): """Apply gsutil-specific patches to Boto. Here be dragons. Sorry. Note that this method should not be used as a replacement for contributing fixes to the upstream Boto library. However, the Boto library has historically not been consistent about release cadence, so upstream fixes may not be available immediately in a version which we can pin to. Also, some fixes may only be applicable to gsutil. In such cases, patches should be applied to the Boto library here (and removed if/when they are included in the upstream repository and included in an official new release that we pull in). This method should be invoked after all other Boto-related initialization has been completed. """ # We have to do all sorts of gross things here (dynamic imports, invalid names # to resolve symbols in copy/pasted methods, invalid spacing from copy/pasted # methods, etc.), so we just disable pylint warnings for this whole method. # yapf: disable # pylint: disable=all # This should have already been imported if this method was called in the # correct place, but we need to perform the import within this method for # this module to recognize it. We don't import this at module level because # its import timing is important and is managed elsewhere. import gcs_oauth2_boto_plugin # TODO(https://github.com/boto/boto/issues/3831): Remove this if the GitHub # issue is ever addressed. orig_get_plugin_method = boto.plugin.get_plugin def _PatchedGetPluginMethod(cls, requested_capability=None): handler_subclasses = orig_get_plugin_method( cls, requested_capability=requested_capability) # In Boto's logic, higher precedence handlers should be closer to the end # of the list. We always want to prefer OAuth2 credentials over HMAC # credentials if both are present, so we shuffle OAuth2 handlers to the end # of the handler list. xml_oauth2_handlers = ( gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2ServiceAccountAuth, gcs_oauth2_boto_plugin.oauth2_plugin.OAuth2Auth) new_result = ( # We need to sort each of these to avoid inconsistent handler selection # when multiple credential types are configured in Python 3.5. See # https://issuetracker.google.com/issues/135709541 for specific logs. sorted( [r for r in handler_subclasses if r not in xml_oauth2_handlers], # Types aren't sortable, so we use their names: key=(lambda handler_t: handler_t.__name__), ) + # Now append XML handlers to the end (highest precedence) sorted( [r for r in handler_subclasses if r in xml_oauth2_handlers], key=(lambda handler_t: handler_t.__name__), )) return new_result boto.plugin.get_plugin = _PatchedGetPluginMethod ######################################################################### # TODO(boto>2.49.0): Remove this. # Fixes SSL issue where SNI was not being used for OpenSSL 1.1.1+ # https://github.com/boto/boto/pull/3843/files # Import modules and resolve symbols used by our patch method. import socket import ssl InvalidCertificateException = ( boto.https_connection.InvalidCertificateException) ValidateCertificateHostname = ( boto.https_connection.ValidateCertificateHostname) def _PatchedConnectMethod(self): # The lines below were copied directly from the Boto file, so we don't lint # or otherwise alter them. if hasattr(self, "timeout"): sock = socket.create_connection((self.host, self.port), self.timeout) else: sock = socket.create_connection((self.host, self.port)) msg = "wrapping ssl socket; " if self.ca_certs: msg += "CA certificate file=%s" % self.ca_certs else: msg += "using system provided SSL certs" boto.log.debug(msg) if hasattr(ssl, 'SSLContext') and getattr(ssl, 'HAS_SNI', False): # Use SSLContext so we can specify server_hostname for SNI # (Required for connections to storage.googleapis.com) context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_REQUIRED if self.ca_certs: context.load_verify_locations(self.ca_certs) if self.cert_file: context.load_cert_chain(self.cert_file, self.key_file) self.sock = context.wrap_socket(sock, server_hostname=self.host) # Add attributes only set in SSLSocket constructor without context: self.sock.keyfile = self.key_file self.sock.certfile = self.cert_file self.sock.cert_reqs = context.verify_mode self.sock.ssl_version = ssl.PROTOCOL_SSLv23 self.sock.ca_certs = self.ca_certs self.sock.ciphers = None else: self.sock = ssl.wrap_socket(sock, keyfile=self.key_file, certfile=self.cert_file, cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.ca_certs) cert = self.sock.getpeercert() hostname = self.host.split(':', 0)[0] if not ValidateCertificateHostname(cert, hostname): raise InvalidCertificateException( hostname, cert, 'remote hostname "%s" does not match ' 'certificate' % hostname) # End `_PatchedConnectMethod` declaration. boto.https_connection.CertValidatingHTTPSConnection.connect = ( _PatchedConnectMethod) def ProxyInfoFromEnvironmentVar(proxy_env_var): """Reads proxy info from the environment and converts to httplib2.ProxyInfo. Args: proxy_env_var: Environment variable string to read, such as http_proxy or https_proxy. Returns: httplib2.ProxyInfo constructed from the environment string. """ proxy_url = os.environ.get(proxy_env_var) if not proxy_url or not proxy_env_var.lower().startswith('http'): return httplib2.ProxyInfo(httplib2.socks.PROXY_TYPE_HTTP, None, 0) proxy_protocol = proxy_env_var.lower().split('_')[0] if not proxy_url.lower().startswith('http'): # proxy_info_from_url requires a protocol, which is always http or https. proxy_url = proxy_protocol + '://' + proxy_url return httplib2.proxy_info_from_url(proxy_url, method=proxy_protocol) def ResumableThreshold(): return config.getint('GSUtil', 'resumable_threshold', 8 ONE_MIB) def SetProxyInfo(boto_proxy_config): """Sets proxy info from boto and environment and converts to httplib2.ProxyInfo. Args: dict: Values read from the .boto file Returns: httplib2.ProxyInfo constructed from boto or environment variable string. """ #Defining proxy_type based on httplib2 library, accounting for None entry too. proxy_type_spec = {'socks4': 1, 'socks5': 2, 'http': 3, 'https': 3} #proxy_type defaults to 'http (3)' for backwards compatibility proxy_type = proxy_type_spec.get( boto_proxy_config.get('proxy_type').lower(), proxy_type_spec['http']) proxy_host = boto_proxy_config.get('proxy_host') proxy_port = boto_proxy_config.get('proxy_port') proxy_user = boto_proxy_config.get('proxy_user') proxy_pass = boto_proxy_config.get('proxy_pass') proxy_rdns = bool(boto_proxy_config.get('proxy_rdns')) #For proxy_info below, proxy_rdns fails for socks4 and socks5 so restricting use #to http only proxy_info = httplib2.ProxyInfo(proxy_host=proxy_host, proxy_type=proxy_type, proxy_port=proxy_port, proxy_user=proxy_user, proxy_pass=proxy_pass, proxy_rdns=proxy_rdns) #Added to force socks proxies not to use rdns if not (proxy_info.proxy_type == proxy_type_spec['http']): proxy_info.proxy_rdns = False if not (proxy_info.proxy_host and proxy_info.proxy_port): # Fall back to using the environment variable. Use only http proxies. for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var) # Assume proxy_rnds is True if a proxy environment variable exists. if boto_proxy_config.get('proxy_rdns') == None: proxy_info.proxy_rdns = True break return proxy_info def UsingCrcmodExtension(): boto_opt = boto.config.get('GSUtil', 'test_assume_fast_crcmod', None) if boto_opt is not None: return boto_opt # Python 3 makes this attribute tough to access due to the way the top-level # crcmod package imports and (identically) names its crcmod module. The only # way to get it is "from crcmod.crcmod import _usingExtension". This is the # alternative form of that statement, but doesn't pollute this module's # namespace with a "_usingExtension" attribute. This also works in both Python # 2.7 and 3.5+. nested_crcmod = __import__( 'crcmod.crcmod', globals(), locals(), ['_usingExtension'], 0, ) return getattr(nested_crcmod, '_usingExtension', False) # TODO(boto-2.49.0): Remove when we pull in the next version of Boto. def _PatchedShouldRetryMethod(self, response, chunked_transfer=False): """Replaces boto.s3.key's should_retry() to handle KMS-encrypted objects.""" # We copy/pasted this from boto and slightly modified it; keep the old # formatting style. # yapf: disable provider = self.bucket.connection.provider if not chunked_transfer: if response.status in [500, 503]: # 500 & 503 can be plain retries. return True if response.getheader('location'): # If there's a redirect, plain retry. return True if 200 <= response.status <= 299: self.etag = response.getheader('etag') md5 = self.md5 if isinstance(md5, bytes): md5 = md5.decode(UTF8) # If you use customer-provided encryption keys, the ETag value that # Amazon S3 returns in the response will not be the MD5 of the # object. amz_server_side_encryption_customer_algorithm = response.getheader( 'x-amz-server-side-encryption-customer-algorithm', None) # The same is applicable for KMS-encrypted objects in gs buckets. goog_customer_managed_encryption = response.getheader( 'x-goog-encryption-kms-key-name', None) if (amz_server_side_encryption_customer_algorithm is None and goog_customer_managed_encryption is None): if self.etag != '"%s"' % md5: raise provider.storage_data_error( 'ETag from S3 did not match computed MD5. ' '%s vs. %s' % (self.etag, self.md5)) return True if response.status == 400: # The 400 must be trapped so the retry handler can check to # see if it was a timeout. # If ``RequestTimeout`` is present, we'll retry. Otherwise, bomb # out. body = response.read() err = provider.storage_response_error( response.status, response.reason, body ) if err.error_code in ['RequestTimeout']: raise boto.exception.PleaseRetryException( "Saw %s, retrying" % err.error_code, response=response ) return False def HasUserSpecifiedGsHost(): gs_host = boto.config.get('Credentials', 'gs_host', None) default_host = six.ensure_str(boto.gs.connection.GSConnection.DefaultHost) if gs_host is not None: return default_host == six.ensure_str(gs_host) return False def UsingGsHmac(): # NOTE: External credentials are omitted intentionally as HMAC takes priority # over the external credentials types. config = boto.config has_refresh_token = config.has_option('Credentials', 'gs_oauth2_refresh_token') has_service_account_credentials = ( config.has_option('Credentials', 'gs_service_client_id') and config.has_option('Credentials', 'gs_service_key_file')) has_hmac_credentials = ( config.has_option('Credentials', 'gs_access_key_id') and config.has_option('Credentials', 'gs_secret_access_key')) return (not has_refresh_token and not has_service_account_credentials and has_hmac_credentials)	26.01 KB	2025-06-25 08:19
📄 cat_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper for cat and cp streaming download.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import io import sys from boto import config from gslib.cloud_api import EncryptionException from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.storage_url import StorageUrlFromString from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils import text_util _CAT_BUCKET_LISTING_FIELDS = [ 'bucket', 'contentEncoding', 'crc32c', 'customerEncryption', 'generation', 'md5Hash', 'name', 'size', ] class CatHelper(object): """Provides methods for the "cat" command and associated functionality.""" def __init__(self, command_obj): """Initializes the helper object. Args: command_obj: gsutil command instance of calling command. """ self.command_obj = command_obj def _WriteBytesBufferedFileToFile(self, src_fd, dst_fd): """Copies contents of the source to the destination via buffered IO. Buffered reads are necessary in the case where you're reading from a source that produces more data than can fit into memory all at once. This method does not close either file when finished. Args: src_fd: The already-open source file to read from. dst_fd: The already-open destination file to write to. """ while True: buf = src_fd.read(io.DEFAULT_BUFFER_SIZE) if not buf: break text_util.write_to_fd(dst_fd, buf) def CatUrlStrings(self, url_strings, show_header=False, start_byte=0, end_byte=None, cat_out_fd=None): """Prints each of the url strings to stdout. Args: url_strings: String iterable. show_header: If true, print a header per file. start_byte: Starting byte of the file to print, used for constructing range requests. end_byte: Ending byte of the file to print; used for constructing range requests. If this is negative, the start_byte is ignored and and end range is sent over HTTP (such as range: bytes -9) cat_out_fd: File descriptor to which output should be written. Defaults to stdout if no file descriptor is supplied. Returns: 0 on success. Raises: CommandException if no URLs can be found. """ printed_one = False # This should refer to whatever sys.stdin refers to when this method is # run, not when this method is defined, so we do the initialization here # rather than define sys.stdin as the cat_out_fd parameter's default value. if cat_out_fd is None: cat_out_fd = sys.stdout # We manipulate the stdout so that all other data other than the Object # contents go to stderr. old_stdout = sys.stdout sys.stdout = sys.stderr try: if url_strings and url_strings[0] in ('-', 'file://-'): self._WriteBytesBufferedFileToFile(sys.stdin, cat_out_fd) else: for url_str in url_strings: did_some_work = False # TODO: Get only the needed fields here. for blr in self.command_obj.WildcardIterator(url_str).IterObjects( bucket_listing_fields=_CAT_BUCKET_LISTING_FIELDS): decryption_keywrapper = None if (blr.root_object and blr.root_object.customerEncryption and blr.root_object.customerEncryption.keySha256): decryption_key = FindMatchingCSEKInBotoConfig( blr.root_object.customerEncryption.keySha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption ' 'key matches object %s' % (blr.root_object.customerEncryption.keySha256, blr.url_string)) decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) did_some_work = True if show_header: if printed_one: print() print('==> %s <==' % blr) printed_one = True cat_object = blr.root_object # This if statement ensures nothing is outputted and no error # is thrown if the user enters an out of bounds range for the object. if 0 < getattr(cat_object, 'size', -1) <= start_byte: return 0 storage_url = StorageUrlFromString(blr.url_string) if storage_url.IsCloudUrl(): compressed_encoding = ObjectIsGzipEncoded(cat_object) self.command_obj.gsutil_api.GetObjectMedia( cat_object.bucket, cat_object.name, cat_out_fd, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, object_size=cat_object.size, generation=storage_url.generation, decryption_tuple=decryption_keywrapper, provider=storage_url.scheme) cat_out_fd.flush() else: with open(storage_url.object_name, 'rb') as f: self._WriteBytesBufferedFileToFile(f, cat_out_fd) if not did_some_work: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) finally: sys.stdout = old_stdout return 0	6.22 KB	2025-06-25 08:19
📄 cloud_api_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for Cloud API implementations.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import re import six from gslib.cloud_api import ArgumentException from gslib.utils.text_util import AddQueryParamToUrl def GetCloudApiInstance(cls, thread_state=None): """Gets a gsutil Cloud API instance. Since Cloud API implementations are not guaranteed to be thread-safe, each thread needs its own instance. These instances are passed to each thread via the thread pool logic in command. Args: cls: Command class to be used for single-threaded case. thread_state: Per thread state from this thread containing a gsutil Cloud API instance. Returns: gsutil Cloud API instance. """ return thread_state or cls.gsutil_api def GetDownloadSerializationData(src_obj_metadata, progress=0, user_project=None): """Returns download serialization data. There are five entries: auto_transfer: JSON-specific field, always False. progress: How much of the download has already been completed. total_size: Total object size. url: Implementation-specific field used for saving a metadata get call. For JSON, this the download URL of the object. For XML, this is a pickled boto key. user_project: Project to be billed to, added as query param. Args: src_obj_metadata: Object to be downloaded. progress: See above. user_project: User project to add to query string. Returns: Serialization data for use with Cloud API GetObjectMedia. """ url = src_obj_metadata.mediaLink if user_project: url = AddQueryParamToUrl(url, 'userProject', user_project) if six.PY3: if isinstance(url, bytes): url = url.decode('ascii') serialization_dict = { 'auto_transfer': 'False', 'progress': progress, 'total_size': src_obj_metadata.size, 'url': url } return json.dumps(serialization_dict) def ListToGetFields(list_fields=None): """Removes 'items/' from the input fields and converts it to a set. Args: list_fields: Iterable fields usable in ListBuckets/ListObjects calls. Returns: Set of fields usable in GetBucket/GetObjectMetadata calls (None implies all fields should be returned). """ if list_fields: get_fields = set() for field in list_fields: if field in ('kind', 'nextPageToken', 'prefixes'): # These are not actually object / bucket metadata fields. # They are fields specific to listing, so we don't consider them. continue get_fields.add(re.sub(r'items/', '', field)) return get_fields def ValidateDstObjectMetadata(dst_obj_metadata): """Ensures dst_obj_metadata supplies the needed fields for copy and insert. Args: dst_obj_metadata: Metadata to validate. Raises: ArgumentException if metadata is invalid. """ if not dst_obj_metadata: raise ArgumentException( 'No object metadata supplied for destination object.') if not dst_obj_metadata.name: raise ArgumentException( 'Object metadata supplied for destination object had no object name.') if not dst_obj_metadata.bucket: raise ArgumentException( 'Object metadata supplied for destination object had no bucket name.')	3.96 KB	2025-06-25 08:19
📄 constants.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared, hard-coded constants. A constant should not be placed in this file if: - it requires complicated or conditional logic to initialize. - it requires importing any modules outside of the Python standard library. This helps reduce dependency graph complexity and the chance of cyclic deps. - it is only used in one file (in which case it should be defined within that module). - it semantically belongs somewhere else (e.g. 'BYTES_PER_KIB' would belong in unit_util.py). """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import sys import six from gslib.utils.unit_util import ONE_GIB from gslib.utils.unit_util import ONE_KIB from gslib.utils.unit_util import ONE_MIB # Readable descriptions for httplib2 logging levels. DEBUGLEVEL_DUMP_REQUESTS = 3 DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS = 4 DEFAULT_FILE_BUFFER_SIZE = 8 * ONE_KIB DEFAULT_GCS_JSON_API_VERSION = 'v1' DEFAULT_GSUTIL_STATE_DIR = os.path.expanduser(os.path.join('~', '.gsutil')) GSUTIL_PUB_TARBALL = 'gs://pub/gsutil.tar.gz' GSUTIL_PUB_TARBALL_PY2 = 'gs://pub/gsutil4.tar.gz' IAM_POLICY_VERSION = 3 IMPERSONATE_SERVICE_ACCOUNT = '' # Number of seconds to wait before printing a long retry warning message. LONG_RETRY_WARN_SEC = 10 # Compressed transport encoded uploads buffer chunks of compressed data. When # running many uploads in parallel, compression may consume more memory than # available. This restricts the number of compressed transport encoded uploads # running in parallel such that they don't consume more memory than set here. MAX_UPLOAD_COMPRESSION_BUFFER_SIZE = 2 * ONE_GIB # On Unix-like systems, we will set the maximum number of open files to avoid # hitting the limit imposed by the OS. This number was obtained experimentally. MIN_ACCEPTABLE_OPEN_FILES_LIMIT = 1000 # For files >= this size, output a message indicating that we're running an # operation on the file (like hashing or gzipping) so it does not appear to the # user that the command is hanging. # TODO: This should say the unit in the name. MIN_SIZE_COMPUTE_LOGGING = 100 * ONE_MIB # The way NO_MAX is used, what is really needed here is the maximum container # size in the Python C code, so using six.MAXSIZE which provides that portably. NO_MAX = six.MAXSIZE # Number of objects to request in listing calls. NUM_OBJECTS_PER_LIST_PAGE = 1000 RELEASE_NOTES_URL = 'https://pub.storage.googleapis.com/gsutil_ReleaseNotes.txt' REQUEST_REASON_ENV_VAR = 'CLOUDSDK_CORE_REQUEST_REASON' REQUEST_REASON_HEADER_KEY = 'x-goog-request-reason' RESUMABLE_THRESHOLD_MIB = 8 RESUMABLE_THRESHOLD_B = RESUMABLE_THRESHOLD_MIB * ONE_MIB # gsutil-specific GUIDs for marking special metadata for S3 compatibility. S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker' S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker' S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID] # By default, the timeout for SSL read errors is infinite. This could # cause gsutil to hang on network disconnect, so pick a more reasonable # timeout. SSL_TIMEOUT_SEC = 60 # Start with a progress callback every <X> KiB during uploads/downloads (JSON # API). Callback implementation should back off until it hits some maximum size # so that callbacks do not create huge amounts of log output. START_CALLBACK_PER_BYTES = 256 * ONE_KIB # Upload/download files in 8 KiB chunks over the HTTP connection. # TODO: This should say the unit in the name. if 'win32' in str(sys.platform).lower(): TRANSFER_BUFFER_SIZE = 64 * ONE_KIB else: TRANSFER_BUFFER_SIZE = 8 * ONE_KIB UTF8 = 'utf-8' WINDOWS_1252 = 'cp1252' # Default number of progress callbacks during transfer (XML API). XML_PROGRESS_CALLBACKS = 10 class Scopes(object): """Enum class for auth scopes, as unicode.""" CLOUD_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform' CLOUD_PLATFORM_READ_ONLY = ( 'https://www.googleapis.com/auth/cloud-platform.read-only') FULL_CONTROL = 'https://www.googleapis.com/auth/devstorage.full_control' READ_ONLY = 'https://www.googleapis.com/auth/devstorage.read_only' READ_WRITE = 'https://www.googleapis.com/auth/devstorage.read_write' REAUTH = 'https://www.googleapis.com/auth/accounts.reauth'	4.83 KB	2025-06-25 08:19
📄 copy_helper.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # Copyright 2011, Nexenta Systems Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for copy functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 from collections import namedtuple import csv import datetime import errno import gzip import json import logging import mimetypes from operator import attrgetter import os import pickle import pyu2f import random import re import shutil import six import stat import subprocess import tempfile import textwrap import time import traceback import six from six.moves import range from six.moves import range from apitools.base.protorpclite import protojson from boto import config import crcmod import gslib from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import CloudApi from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException from gslib.commands.compose import MAX_COMPOSE_ARITY from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE from gslib.commands.config import DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS from gslib.commands.config import DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD from gslib.commands.config import DEFAULT_GZIP_COMPRESSION_LEVEL from gslib.cs_api_map import ApiSelector from gslib.daisy_chain_wrapper import DaisyChainWrapper from gslib.exception import CommandException from gslib.exception import HashMismatchException from gslib.exception import InvalidUrlError from gslib.file_part import FilePart from gslib.parallel_tracker_file import GenerateComponentObjectPrefix from gslib.parallel_tracker_file import ReadParallelUploadTrackerFile from gslib.parallel_tracker_file import ValidateParallelCompositeTrackerData from gslib.parallel_tracker_file import WriteComponentToParallelUploadTrackerFile from gslib.parallel_tracker_file import WriteParallelUploadTrackerFile from gslib.progress_callback import FileProgressCallbackHandler from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.resumable_streaming_upload import ResumableStreamingJsonUploadWrapper from gslib import storage_url from gslib.storage_url import ContainsWildcard from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import IsCloudSubdirPlaceholder from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FileMessage from gslib.thread_message import RetryableErrorMessage from gslib.tracker_file import DeleteDownloadTrackerFiles from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import ENCRYPTION_UPLOAD_TRACKER_ENTRY from gslib.tracker_file import GetDownloadStartByte from gslib.tracker_file import GetTrackerFilePath from gslib.tracker_file import GetUploadTrackerData from gslib.tracker_file import RaiseUnwritableTrackerFileException from gslib.tracker_file import ReadOrCreateDownloadTrackerFile from gslib.tracker_file import SERIALIZATION_UPLOAD_TRACKER_ENTRY from gslib.tracker_file import TrackerFileType from gslib.tracker_file import WriteDownloadComponentTrackerFile from gslib.tracker_file import WriteJsonDataToTrackerFile from gslib.utils import parallelism_framework_util from gslib.utils import stet_util from gslib.utils import temporary_file_util from gslib.utils import text_util from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNumRetries from gslib.utils.boto_util import ResumableThreshold from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.cloud_api_helper import GetCloudApiInstance from gslib.utils.cloud_api_helper import GetDownloadSerializationData from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE from gslib.utils.constants import MIN_SIZE_COMPUTE_LOGGING from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.encryption_helper import GetEncryptionKeyWrapper from gslib.utils.hashing_helper import Base64EncodeHash from gslib.utils.hashing_helper import CalculateB64EncodedMd5FromContents from gslib.utils.hashing_helper import CalculateHashesFromContents from gslib.utils.hashing_helper import CHECK_HASH_IF_FAST_ELSE_FAIL from gslib.utils.hashing_helper import CHECK_HASH_NEVER from gslib.utils.hashing_helper import ConcatCrc32c from gslib.utils.hashing_helper import GetDownloadHashAlgs from gslib.utils.hashing_helper import GetMd5 from gslib.utils.hashing_helper import GetUploadHashAlgs from gslib.utils.hashing_helper import HashingFileUploadWrapper from gslib.utils.metadata_util import ObjectIsGzipEncoded from gslib.utils.parallelism_framework_util import AtomicDict from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.posix_util import ATIME_ATTR from gslib.utils.posix_util import ConvertDatetimeToPOSIX from gslib.utils.posix_util import GID_ATTR from gslib.utils.posix_util import MODE_ATTR from gslib.utils.posix_util import MTIME_ATTR from gslib.utils.posix_util import ParseAndSetPOSIXAttributes from gslib.utils.posix_util import UID_ATTR from gslib.utils.system_util import CheckFreeSpace from gslib.utils.system_util import GetFileSize from gslib.utils.system_util import GetStreamFromFileUrl from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AddS3MarkerAclToObjectMetadata from gslib.utils.translation_helper import CopyObjectMetadata from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE from gslib.utils.translation_helper import ObjectMetadataFromHeaders from gslib.utils.translation_helper import PreconditionsFromHeaders from gslib.utils.translation_helper import S3MarkerAclFromObjectMetadata from gslib.utils.unit_util import DivideAndCeil from gslib.utils.unit_util import HumanReadableToBytes from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.unit_util import TEN_MIB from gslib.wildcard_iterator import CreateWildcardIterator if six.PY3: long = int # pylint: disable=g-import-not-at-top if IS_WINDOWS: import msvcrt # Declare copy_helper_opts as a global because namedtuple isn't aware of # assigning to a class member (which breaks pickling done by multiprocessing). # For details see # http://stackoverflow.com/questions/16377215/how-to-pickle-a-namedtuple-instance-correctly # pylint: disable=global-at-module-level global global_copy_helper_opts # In-memory map of local files that are currently opened for write. Used to # ensure that if we write to the same file twice (say, for example, because the # user specified two identical source URLs), the writes occur serially. global open_files_map, open_files_lock open_files_map = AtomicDict( manager=(parallelism_framework_util.top_level_manager if CheckMultiprocessingAvailableAndInit().is_available else None)) # We don't allow multiple processes on Windows, so using a process-safe lock # would be unnecessary. open_files_lock = parallelism_framework_util.CreateLock() # For debugging purposes; if True, files and objects that fail hash validation # will be saved with the below suffix appended. _RENAME_ON_HASH_MISMATCH = False _RENAME_ON_HASH_MISMATCH_SUFFIX = '_corrupt' PARALLEL_UPLOAD_TEMP_NAMESPACE = ( '/gsutil/tmp/parallel_composite_uploads/for_details_see/gsutil_help_cp/') PARALLEL_UPLOAD_STATIC_SALT = u""" PARALLEL_UPLOAD_SALT_TO_PREVENT_COLLISIONS. The theory is that no user will have prepended this to the front of one of their object names and then done an MD5 hash of the name, and then prepended PARALLEL_UPLOAD_TEMP_NAMESPACE to the front of their object name. Note that there will be no problems with object name length since we hash the original name. """ # When uploading a file, get the following fields in the response for # filling in command output and manifests. UPLOAD_RETURN_FIELDS = [ 'crc32c', 'customerEncryption', 'etag', 'generation', 'md5Hash', 'size', ] # This tuple is used only to encapsulate the arguments needed for # command.Apply() in the parallel composite upload case. # Note that content_type is used instead of a full apitools Object() because # apitools objects are not picklable. # filename: String name of file. # file_start: start byte of file (may be in the middle of a file for partitioned # files). # file_length: length of upload (may not be the entire length of a file for # partitioned files). # src_url: FileUrl describing the source file. # dst_url: CloudUrl describing the destination component file. # canned_acl: canned_acl to apply to the uploaded file/component. # content_type: content-type for final object, used for setting content-type # of components and final object. # tracker_file: tracker file for this component. # tracker_file_lock: tracker file lock for tracker file(s). # gzip_encoded: Whether to use gzip transport encoding for the upload. PerformParallelUploadFileToObjectArgs = namedtuple( 'PerformParallelUploadFileToObjectArgs', 'filename file_start file_length src_url dst_url canned_acl ' 'content_type storage_class tracker_file tracker_file_lock encryption_key_sha256 ' 'gzip_encoded') PerformSlicedDownloadObjectToFileArgs = namedtuple( 'PerformSlicedDownloadObjectToFileArgs', 'component_num src_url src_obj_metadata_json dst_url download_file_name ' 'start_byte end_byte decryption_key') # This tuple is used only to encapsulate the arguments returned by # _PerformSlicedDownloadObjectToFile. # component_num: Component number. # crc32c: CRC32C hash value (integer) of the downloaded bytes # bytes_transferred: The number of bytes transferred, potentially less # than the component size if the download was resumed. # component_total_size: The number of bytes corresponding to the whole # component size, potentially more than bytes_transferred # if the download was resumed. # server_encoding: Content-encoding string if it was detected that the server # sent encoded bytes during transfer, None otherwise. PerformSlicedDownloadReturnValues = namedtuple( 'PerformSlicedDownloadReturnValues', 'component_num crc32c bytes_transferred component_total_size ' 'server_encoding') # TODO: Refactor this file to be less cumbersome. In particular, some of the # different paths (e.g., uploading a file to an object vs. downloading an # object to a file) could be split into separate files. # Chunk size to use while zipping/unzipping gzip files. GZIP_CHUNK_SIZE = 8192 # Indicates that all files should be gzipped, in _UploadFileToObject GZIP_ALL_FILES = 'GZIP_ALL_FILES' # Number of bytes to wait before updating a sliced download component tracker # file. TRACKERFILE_UPDATE_THRESHOLD = TEN_MIB PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD = 150 * 1024 * 1024 # S3 requires special Multipart upload logic (that we currently don't implement) # for files > 5GiB in size. S3_MAX_UPLOAD_SIZE = 5 * 1024 * 1024 * 1024 # TODO: Create a message class that serializes posting this message once # through the UI's global status queue. global suggested_sliced_transfers, suggested_sliced_transfers_lock suggested_sliced_transfers = AtomicDict( manager=(parallelism_framework_util.top_level_manager if CheckMultiprocessingAvailableAndInit().is_available else None)) suggested_sliced_transfers_lock = parallelism_framework_util.CreateLock() COMMON_EXTENSION_RULES = { 'md': 'text/markdown', 'tgz': 'application/gzip', } class FileConcurrencySkipError(Exception): """Raised when skipping a file due to a concurrent, duplicate copy.""" def _RmExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) def _ParallelCopyExceptionHandler(cls, e): """Simple exception handler to allow post-completion status.""" cls.logger.error(str(e)) cls.op_failure_count += 1 cls.logger.debug('\n\nEncountered exception while copying:\n%s\n', traceback.format_exc()) def _PerformParallelUploadFileToObject(cls, args, thread_state=None): """Function argument to Apply for performing parallel composite uploads. Args: cls: Calling Command class. args: PerformParallelUploadFileToObjectArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: StorageUrl representing a successfully uploaded component. """ fp = FilePart(args.filename, args.file_start, args.file_length) gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) with fp: # We take many precautions with the component names that make collisions # effectively impossible. Specifying preconditions will just allow us to # reach a state in which uploads will always fail on retries. preconditions = None # Fill in content type and storage class if one was provided. dst_object_metadata = apitools_messages.Object( name=args.dst_url.object_name, bucket=args.dst_url.bucket_name, contentType=args.content_type, storageClass=args.storage_class) orig_prefer_api = gsutil_api.prefer_api try: if global_copy_helper_opts.canned_acl: # No canned ACL support in JSON, force XML API to be used for # upload/copy operations. gsutil_api.prefer_api = ApiSelector.XML ret = _UploadFileToObject(args.src_url, fp, args.file_length, args.dst_url, dst_object_metadata, preconditions, gsutil_api, cls.logger, cls, _ParallelCopyExceptionHandler, gzip_exts=None, allow_splitting=False, is_component=True, gzip_encoded=args.gzip_encoded) finally: if global_copy_helper_opts.canned_acl: gsutil_api.prefer_api = orig_prefer_api component = ret[2] WriteComponentToParallelUploadTrackerFile( args.tracker_file, args.tracker_file_lock, component, cls.logger, encryption_key_sha256=args.encryption_key_sha256) return ret CopyHelperOpts = namedtuple('CopyHelperOpts', [ 'perform_mv', 'no_clobber', 'daisy_chain', 'read_args_from_stdin', 'print_ver', 'use_manifest', 'preserve_acl', 'canned_acl', 'skip_unsupported_objects', 'test_callback_file', 'dest_storage_class', ]) # pylint: disable=global-variable-undefined def CreateCopyHelperOpts(perform_mv=False, no_clobber=False, daisy_chain=False, read_args_from_stdin=False, print_ver=False, use_manifest=False, preserve_acl=False, canned_acl=None, skip_unsupported_objects=False, test_callback_file=None, dest_storage_class=None): """Creates CopyHelperOpts for passing options to CopyHelper.""" # We create a tuple with union of options needed by CopyHelper and any # copy-related functionality in CpCommand, RsyncCommand, or Command class. global global_copy_helper_opts global_copy_helper_opts = CopyHelperOpts( perform_mv=perform_mv, no_clobber=no_clobber, daisy_chain=daisy_chain, read_args_from_stdin=read_args_from_stdin, print_ver=print_ver, use_manifest=use_manifest, preserve_acl=preserve_acl, canned_acl=canned_acl, skip_unsupported_objects=skip_unsupported_objects, test_callback_file=test_callback_file, dest_storage_class=dest_storage_class) return global_copy_helper_opts # pylint: disable=global-variable-undefined # pylint: disable=global-variable-not-assigned def GetCopyHelperOpts(): """Returns namedtuple holding CopyHelper options.""" global global_copy_helper_opts return global_copy_helper_opts def _SelectDownloadStrategy(dst_url): """Get download strategy based on the destination object. Args: dst_url: Destination StorageUrl. Returns: gsutil Cloud API DownloadStrategy. """ dst_is_special = False if dst_url.IsFileUrl(): # Check explicitly first because os.stat doesn't work on 'nul' in Windows. if dst_url.object_name == os.devnull: dst_is_special = True try: mode = os.stat(dst_url.object_name).st_mode if stat.S_ISCHR(mode): dst_is_special = True except OSError: pass if dst_is_special: return CloudApi.DownloadStrategy.ONE_SHOT else: return CloudApi.DownloadStrategy.RESUMABLE def InsistDstUrlNamesContainer(exp_dst_url, have_existing_dst_container, command_name): """Ensures the destination URL names a container. Acceptable containers include directory, bucket, bucket subdir, and non-existent bucket subdir. Args: exp_dst_url: Wildcard-expanded destination StorageUrl. have_existing_dst_container: bool indicator of whether exp_dst_url names a container (directory, bucket, or existing bucket subdir). command_name: Name of command making call. May not be the same as the calling class's self.command_name in the case of commands implemented atop other commands (like mv command). Raises: CommandException: if the URL being checked does not name a container. """ if ((exp_dst_url.IsFileUrl() and not exp_dst_url.IsDirectory()) or (exp_dst_url.IsCloudUrl() and exp_dst_url.IsBucket() and not have_existing_dst_container)): raise CommandException('Destination URL must name a directory, bucket, ' 'or bucket\nsubdirectory for the multiple ' 'source form of the %s command.' % command_name) def _ShouldTreatDstUrlAsBucketSubDir(have_multiple_srcs, dst_url, have_existing_dest_subdir, src_url_names_container, recursion_requested): """Checks whether dst_url should be treated as a bucket "sub-directory". The decision about whether something constitutes a bucket "sub-directory" depends on whether there are multiple sources in this request and whether there is an existing bucket subdirectory. For example, when running the command: gsutil cp file gs://bucket/abc if there's no existing gs://bucket/abc bucket subdirectory we should copy file to the object gs://bucket/abc. In contrast, if there's an existing gs://bucket/abc bucket subdirectory we should copy file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc exists, when running the command: gsutil cp file1 file2 gs://bucket/abc we should copy file1 to gs://bucket/abc/file1 (and similarly for file2). Finally, for recursive copies, if the source is a container then we should copy to a container as the target. For example, when running the command: gsutil cp -r dir1 gs://bucket/dir2 we should copy the subtree of dir1 to gs://bucket/dir2. Note that we don't disallow naming a bucket "sub-directory" where there's already an object at that URL. For example it's legitimate (albeit confusing) to have an object called gs://bucket/dir and then run the command gsutil cp file1 file2 gs://bucket/dir Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: have_multiple_srcs: Bool indicator of whether this is a multi-source operation. dst_url: StorageUrl to check. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. src_url_names_container: bool indicator of whether the source URL is a container. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. """ if have_existing_dest_subdir: return True if dst_url.IsCloudUrl(): return (have_multiple_srcs or (src_url_names_container and recursion_requested)) def _ShouldTreatDstUrlAsSingleton(src_url_names_container, have_multiple_srcs, have_existing_dest_subdir, dst_url, recursion_requested): """Checks that dst_url names a single file/object after wildcard expansion. It is possible that an object path might name a bucket sub-directory. Args: src_url_names_container: Bool indicator of whether the source for the operation is a container (bucket, bucket subdir, or directory). have_multiple_srcs: Bool indicator of whether this is a multi-source operation. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. dst_url: StorageUrl to check. recursion_requested: True if a recursive operation has been requested. Returns: bool indicator. """ if recursion_requested and src_url_names_container: return False if dst_url.IsFileUrl(): return not dst_url.IsDirectory() else: # dst_url.IsCloudUrl() return (not have_multiple_srcs and not have_existing_dest_subdir and dst_url.IsObject()) def _IsUrlValidParentDir(url): """Returns True if not FileUrl ending in relative path symbols. A URL is invalid if it is a FileUrl and the parent directory of the file is a relative path symbol. Unix will not allow a file itself to be named with a relative path symbol, but one can be the parent. Notably, "../obj" can lead to unexpected behavior at the copy destination. We examine the pre-recursion "url", which might point to "..", to see if the parent is valid. If the user does a recursive copy from a URL, it may not end up the final parent of the copied object. For example, see: "dir/nested_dir/obj". If you ran "cp -r dir gs://bucket" from the parent of "dir", then the "url" arg would be "dir", but "nested_dir" would be the parent of "obj". This actually doesn't matter since recursion won't add relative path symbols to the path. However, we still return if "url" is valid because there are cases where we need to copy every parent directory up to "dir" to prevent file name conflicts. Args: url: StorageUrl before recursion. Returns: Boolean indicating if the "url" is valid as a parent directory. """ if not url.IsFileUrl(): return True url_string_minus_trailing_delimiter = url.versionless_url_string.rstrip( url.delim) _, _, after_last_delimiter = (url_string_minus_trailing_delimiter.rpartition( url.delim)) return after_last_delimiter not in storage_url.RELATIVE_PATH_SYMBOLS and ( after_last_delimiter not in [ url.scheme + '://' + symbol for symbol in storage_url.RELATIVE_PATH_SYMBOLS ]) def ConstructDstUrl(src_url, exp_src_url, src_url_names_container, have_multiple_srcs, has_multiple_top_level_srcs, exp_dst_url, have_existing_dest_subdir, recursion_requested, preserve_posix=False): """Constructs the destination URL for a given exp_src_url/exp_dst_url pair. Uses context-dependent naming rules that mimic Linux cp and mv behavior. Args: src_url: Source StorageUrl to be copied. exp_src_url: Single StorageUrl from wildcard expansion of src_url. src_url_names_container: True if src_url names a container (including the case of a wildcard-named bucket subdir (like gs://bucket/abc, where gs://bucket/abc/* matched some objects). have_multiple_srcs: True if this is a multi-source request. This can be true if src_url wildcard-expanded to multiple URLs or if there were multiple source URLs in the request. has_multiple_top_level_srcs: Same as have_multiple_srcs but measured before recursion. exp_dst_url: the expanded StorageUrl requested for the cp destination. Final written path is constructed from this plus a context-dependent variant of src_url. have_existing_dest_subdir: bool indicator whether dest is an existing subdirectory. recursion_requested: True if a recursive operation has been requested. preserve_posix: True if preservation of posix attributes has been requested. Returns: StorageUrl to use for copy. Raises: CommandException if destination object name not specified for source and source is a stream. """ if (exp_dst_url.IsFileUrl() and exp_dst_url.IsStream() and preserve_posix): raise CommandException('Cannot preserve POSIX attributes with a stream.') if _ShouldTreatDstUrlAsSingleton(src_url_names_container, have_multiple_srcs, have_existing_dest_subdir, exp_dst_url, recursion_requested): # We're copying one file or object to one file or object. return exp_dst_url if exp_src_url.IsFileUrl() and (exp_src_url.IsStream() or exp_src_url.IsFifo()): if have_existing_dest_subdir: type_text = 'stream' if exp_src_url.IsStream() else 'named pipe' raise CommandException('Destination object name needed when ' 'source is a %s' % type_text) return exp_dst_url if not recursion_requested and not have_multiple_srcs: # We're copying one file or object to a subdirectory. Append final comp # of exp_src_url to exp_dst_url. src_final_comp = exp_src_url.object_name.rpartition(src_url.delim)[-1] return StorageUrlFromString('%s%s%s' % (exp_dst_url.url_string.rstrip( exp_dst_url.delim), exp_dst_url.delim, src_final_comp)) # Else we're copying multiple sources to a directory, bucket, or a bucket # "sub-directory". # Ensure exp_dst_url ends in delim char if we're doing a multi-src copy or # a copy to a directory. (The check for copying to a directory needs # special-case handling so that the command: # gsutil cp gs://bucket/obj dir # will turn into file://dir/ instead of file://dir -- the latter would cause # the file "dirobj" to be created.) # Note: need to check have_multiple_srcs or src_url.names_container() # because src_url could be a bucket containing a single object, named # as gs://bucket. if ((have_multiple_srcs or src_url_names_container or (exp_dst_url.IsFileUrl() and exp_dst_url.IsDirectory())) and not exp_dst_url.url_string.endswith(exp_dst_url.delim)): exp_dst_url = StorageUrlFromString( '%s%s' % (exp_dst_url.url_string, exp_dst_url.delim)) src_url_is_valid_parent = _IsUrlValidParentDir(src_url) if not src_url_is_valid_parent and has_multiple_top_level_srcs: # To avoid top-level name conflicts, we need to copy the parent dir. # However, that cannot be done because the parent dir has an invalid name. raise InvalidUrlError( 'Presence of multiple top-level sources and invalid expanded URL' ' make file name conflicts possible for URL: {}'.format(src_url)) # Making naming behavior match how things work with local Linux cp and mv # operations depends on many factors, including whether the destination is a # container, and the plurality of the source(s). # 1. Recursively copying from directories, buckets, or bucket subdirs should # result in objects/files mirroring the source hierarchy. For example: # gsutil cp -r dir1/dir2 gs://bucket # should create the object gs://bucket/dir2/file2, assuming dir1/dir2 # contains file2). # # To be consistent with Linux cp behavior, there's one more wrinkle when # working with subdirs: The resulting object names depend on whether the # destination subdirectory exists. For example, if gs://bucket/subdir # exists, the command: # gsutil cp -r dir1/dir2 gs://bucket/subdir # should create objects named like gs://bucket/subdir/dir2/a/b/c. In # contrast, if gs://bucket/subdir does not exist, this same command # should create objects named like gs://bucket/subdir/a/b/c. # # If there are multiple top-level source items, preserve source parent # dirs. This is similar to when the destination dir already exists and # avoids conflicts such as "dir1/f.txt" and "dir2/f.txt" both getting # copied to "gs://bucket/f.txt". Linux normally errors on these conflicts, # but we cannot do that because we need to give users the ability to create # dirs as they copy to the cloud. # # Note: "mv" is similar to running "cp -r" followed by source deletion. # # 2. Copying individual files or objects to dirs, buckets or bucket subdirs # should result in objects/files named by the final source file name # component. Example: # gsutil cp dir1/.txt gs://bucket # should create the objects gs://bucket/f1.txt and gs://bucket/f2.txt, # assuming dir1 contains f1.txt and f2.txt. # Ignore the "multiple top-level sources" rule if using double wildcard * # because that treats all files as top-level, in which case the user doesn't # want to preserve directories. preserve_src_top_level_dirs = ('*' not in src_url.versionless_url_string and src_url_is_valid_parent and (has_multiple_top_level_srcs or have_existing_dest_subdir)) if preserve_src_top_level_dirs or (src_url_names_container and (exp_dst_url.IsCloudUrl() or exp_dst_url.IsDirectory())): # Case 1. Container copy to a destination other than a file. # Build dst_key_name from subpath of exp_src_url past # where src_url ends. For example, for src_url=gs://bucket/ and # exp_src_url=gs://bucket/src_subdir/obj, dst_key_name should be # src_subdir/obj. src_url_path_sans_final_dir = GetPathBeforeFinalDir(src_url, exp_src_url) dst_key_name = exp_src_url.versionless_url_string[ len(src_url_path_sans_final_dir):].lstrip(src_url.delim) if not preserve_src_top_level_dirs: # Only copy file name, not parent dir. dst_key_name = dst_key_name.partition(src_url.delim)[-1] else: # Case 2. dst_key_name = exp_src_url.object_name.rpartition(src_url.delim)[-1] if (exp_dst_url.IsFileUrl() or _ShouldTreatDstUrlAsBucketSubDir( have_multiple_srcs, exp_dst_url, have_existing_dest_subdir, src_url_names_container, recursion_requested)): if exp_dst_url.object_name and exp_dst_url.object_name.endswith( exp_dst_url.delim): dst_key_name = '%s%s%s' % (exp_dst_url.object_name.rstrip( exp_dst_url.delim), exp_dst_url.delim, dst_key_name) else: delim = exp_dst_url.delim if exp_dst_url.object_name else '' dst_key_name = '%s%s%s' % (exp_dst_url.object_name or '', delim, dst_key_name) new_exp_dst_url = exp_dst_url.Clone() new_exp_dst_url.object_name = dst_key_name.replace(src_url.delim, exp_dst_url.delim) return new_exp_dst_url def _CreateDigestsFromDigesters(digesters): digests = {} if digesters: for alg in digesters: digests[alg] = base64.b64encode( digesters[alg].digest()).rstrip(b'\n').decode('ascii') return digests def _CreateDigestsFromLocalFile(status_queue, algs, file_name, src_url, src_obj_metadata): """Creates a base64 CRC32C and/or MD5 digest from file_name. Args: status_queue: Queue for posting progress messages for UI/Analytics. algs: List of algorithms to compute. file_name: File to digest. src_url: StorageUrl for local object. Used to track progress. src_obj_metadata: Metadata of source object. Returns: Dict of algorithm name : base 64 encoded digest """ hash_dict = {} if 'md5' in algs: hash_dict['md5'] = GetMd5() if 'crc32c' in algs: hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c') with open(file_name, 'rb') as fp: CalculateHashesFromContents(fp, hash_dict, callback_processor=ProgressCallbackWithTimeout( src_obj_metadata.size, FileProgressCallbackHandler( status_queue, src_url=src_url, operation_name='Hashing').call)) digests = {} for alg_name, digest in six.iteritems(hash_dict): digests[alg_name] = Base64EncodeHash(digest.hexdigest()) return digests def _CheckCloudHashes(logger, src_url, dst_url, src_obj_metadata, dst_obj_metadata): """Validates integrity of two cloud objects copied via daisy-chain. Args: logger: for outputting log messages. src_url: CloudUrl for source cloud object. dst_url: CloudUrl for destination cloud object. src_obj_metadata: Cloud Object metadata for object being downloaded from. dst_obj_metadata: Cloud Object metadata for object being uploaded to. Raises: CommandException: if cloud digests don't match local digests. """ # See hack comment in _CheckHashes. # Sometimes (e.g. when kms is enabled for s3) the values we check below are # not actually content hashes. The early exit here provides users a workaround # for this case and any others we've missed. check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == CHECK_HASH_NEVER: return checked_one = False download_hashes = {} upload_hashes = {} if src_obj_metadata.md5Hash: src_md5hash = six.ensure_binary(src_obj_metadata.md5Hash) download_hashes['md5'] = src_md5hash if src_obj_metadata.crc32c: src_crc32c_hash = six.ensure_binary(src_obj_metadata.crc32c) download_hashes['crc32c'] = src_crc32c_hash if dst_obj_metadata.md5Hash: dst_md5hash = six.ensure_binary(dst_obj_metadata.md5Hash) upload_hashes['md5'] = dst_md5hash if dst_obj_metadata.crc32c: dst_crc32c_hash = six.ensure_binary(dst_obj_metadata.crc32c) upload_hashes['crc32c'] = dst_crc32c_hash for alg, upload_b64_digest in six.iteritems(upload_hashes): if alg not in download_hashes: continue download_b64_digest = download_hashes[alg] if six.PY3 and isinstance(download_b64_digest, str): download_b64_digest = download_b64_digest.encode('ascii') logger.debug('Comparing source vs destination %s-checksum for %s. (%s/%s)', alg, dst_url, download_b64_digest, upload_b64_digest) if download_b64_digest != upload_b64_digest: raise HashMismatchException( '%s signature for source object (%s) doesn\'t match ' 'destination object digest (%s). Object (%s) will be deleted.' % (alg, download_b64_digest, upload_b64_digest, dst_url)) checked_one = True if not checked_one: # One known way this can currently happen is when downloading objects larger # than 5 GiB from S3 (for which the etag is not an MD5). logger.warn( 'WARNING: Found no hashes to validate object downloaded from %s and ' 'uploaded to %s. Integrity cannot be assured without hashes.', src_url, dst_url) def _CheckHashes(logger, obj_url, obj_metadata, file_name, digests, is_upload=False): """Validates integrity by comparing cloud digest to local digest. Args: logger: for outputting log messages. obj_url: CloudUrl for cloud object. obj_metadata: Cloud Object being downloaded from or uploaded to. file_name: Local file name on disk being downloaded to or uploaded from (used only for logging). digests: Computed Digests for the object. is_upload: If true, comparing for an uploaded object (controls logging). Raises: CommandException: if cloud digests don't match local digests. """ # Hack below. # I cannot track down all of the code paths that get here, so I finally # gave up and opted to convert all of the hashes to str. I know that they # should* be bytes, but the path of least resistance led to str. # Not a nice thing, but for now it makes tests pass... # Update: Since the _CheckCloudHashes function above needs to be changed # as well, I am going to make the executive decision that hashes are # bytes - here as well. It's what the hash and base64 PY3 libs return, # and should be the native format for these things. local_hashes = digests cloud_hashes = {} if obj_metadata.md5Hash: md5_b64_digest = six.ensure_binary(obj_metadata.md5Hash) cloud_hashes['md5'] = md5_b64_digest.rstrip(b'\n') if obj_metadata.crc32c: crc32c_b64_hash = six.ensure_binary(obj_metadata.crc32c) cloud_hashes['crc32c'] = crc32c_b64_hash.rstrip(b'\n') checked_one = False for alg in local_hashes: if alg not in cloud_hashes: continue local_b64_digest = six.ensure_binary(local_hashes[alg]) cloud_b64_digest = cloud_hashes[alg] logger.debug('Comparing local vs cloud %s-checksum for %s. (%s/%s)', alg, file_name, local_b64_digest, cloud_b64_digest) if local_b64_digest != cloud_b64_digest: raise HashMismatchException( '%s signature computed for local file (%s) doesn\'t match ' 'cloud-supplied digest (%s). %s (%s) will be deleted.' % (alg, local_b64_digest, cloud_b64_digest, 'Cloud object' if is_upload else 'Local file', obj_url if is_upload else file_name)) checked_one = True if not checked_one: if is_upload: logger.warn( 'WARNING: Found no hashes to validate object uploaded to %s. ' 'Integrity cannot be assured without hashes.', obj_url) else: # One known way this can currently happen is when downloading objects larger # than 5 GB from S3 (for which the etag is not an MD5). logger.warn( 'WARNING: Found no hashes to validate object downloaded to %s. ' 'Integrity cannot be assured without hashes.', file_name) def IsNoClobberServerException(e): """Checks to see if the server attempted to clobber a file. In this case we specified via a precondition that we didn't want the file clobbered. Args: e: The Exception that was generated by a failed copy operation Returns: bool indicator - True indicates that the server did attempt to clobber an existing file. """ return ((isinstance(e, PreconditionException)) or (isinstance(e, ResumableUploadException) and '412' in e.message)) def CheckForDirFileConflict(exp_src_url, dst_url): """Checks whether copying exp_src_url into dst_url is not possible. This happens if a directory exists in local file system where a file needs to go or vice versa. In that case we print an error message and exits. Example: if the file "./x" exists and you try to do: gsutil cp gs://mybucket/x/y . the request can't succeed because it requires a directory where the file x exists. Note that we don't enforce any corresponding restrictions for buckets, because the flat namespace semantics for buckets doesn't prohibit such cases the way hierarchical file systems do. For example, if a bucket contains an object called gs://bucket/dir and then you run the command: gsutil cp file1 file2 gs://bucket/dir you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and gs://bucket/dir/file2. Args: exp_src_url: Expanded source StorageUrl. dst_url: Destination StorageUrl. Raises: CommandException: if errors encountered. """ if dst_url.IsCloudUrl(): # The problem can only happen for file destination URLs. return dst_path = dst_url.object_name final_dir = os.path.dirname(dst_path) if os.path.isfile(final_dir): raise CommandException('Cannot retrieve %s because a file exists ' 'where a directory needs to be created (%s).' % (exp_src_url.url_string, final_dir)) if os.path.isdir(dst_path): raise CommandException('Cannot retrieve %s because a directory exists ' '(%s) where the file needs to be created.' % (exp_src_url.url_string, dst_path)) def _PartitionFile(canned_acl, content_type, dst_bucket_url, file_size, fp, random_prefix, src_url, storage_class, tracker_file, tracker_file_lock, encryption_key_sha256=None, gzip_encoded=False): """Partitions a file into FilePart objects to be uploaded and later composed. These objects, when composed, will match the original file. This entails splitting the file into parts, naming and forming a destination URL for each part, and also providing the PerformParallelUploadFileToObjectArgs corresponding to each part. Args: canned_acl: The user-provided canned_acl, if applicable. content_type: content type for the component and final objects. dst_bucket_url: CloudUrl for the destination bucket. file_size: The size of fp, in bytes. fp: The file object to be partitioned. random_prefix: The randomly-generated prefix used to prevent collisions among the temporary component names. src_url: Source FileUrl from the original command. storage_class: storage class for the component and final objects. tracker_file: The path to the parallel composite upload tracker file. tracker_file_lock: The lock protecting access to the tracker file. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: dst_args: The destination URIs for the temporary component objects. """ parallel_composite_upload_component_size = HumanReadableToBytes( config.get('GSUtil', 'parallel_composite_upload_component_size', DEFAULT_PARALLEL_COMPOSITE_UPLOAD_COMPONENT_SIZE)) (num_components, component_size) = _GetPartitionInfo( file_size, MAX_COMPOSE_ARITY, parallel_composite_upload_component_size) dst_args = {} # Arguments to create commands and pass to subprocesses. file_names = [] # Used for the 2-step process of forming dst_args. for i in range(num_components): # "Salt" the object name with something a user is very unlikely to have # used in an object name, then hash the extended name to make sure # we don't run into problems with name length. Using a deterministic # naming scheme for the temporary components allows users to take # advantage of resumable uploads for each component. encoded_name = six.ensure_binary(PARALLEL_UPLOAD_STATIC_SALT + fp.name) content_md5 = GetMd5() content_md5.update(encoded_name) digest = content_md5.hexdigest() temp_file_name = (random_prefix + PARALLEL_UPLOAD_TEMP_NAMESPACE + digest + '_' + str(i)) tmp_dst_url = dst_bucket_url.Clone() tmp_dst_url.object_name = temp_file_name if i < (num_components - 1): # Every component except possibly the last is the same size. file_part_length = component_size else: # The last component just gets all of the remaining bytes. file_part_length = (file_size - ((num_components - 1) * component_size)) offset = i * component_size func_args = PerformParallelUploadFileToObjectArgs( fp.name, offset, file_part_length, src_url, tmp_dst_url, canned_acl, content_type, storage_class, tracker_file, tracker_file_lock, encryption_key_sha256, gzip_encoded) file_names.append(temp_file_name) dst_args[temp_file_name] = func_args return dst_args def _GetComponentNumber(component): """Gets component number from component CloudUrl. Used during parallel composite upload. Args: component: CloudUrl representing component. Returns: component number """ return int(component.object_name[component.object_name.rfind('_') + 1:]) def _DoParallelCompositeUpload(fp, src_url, dst_url, dst_obj_metadata, canned_acl, file_size, preconditions, gsutil_api, command_obj, copy_exception_handler, logger, gzip_encoded=False): """Uploads a local file to a cloud object using parallel composite upload. The file is partitioned into parts, and then the parts are uploaded in parallel, composed to form the original destination object, and deleted. Args: fp: The file object to be uploaded. src_url: FileUrl representing the local file. dst_url: CloudUrl representing the destination file. dst_obj_metadata: apitools Object describing the destination object. canned_acl: The canned acl to apply to the object, if any. file_size: The size of the source file in bytes. preconditions: Cloud API Preconditions for the final object. gsutil_api: gsutil Cloud API instance to use. command_obj: Command object (for calling Apply). copy_exception_handler: Copy exception handler (for use in Apply). logger: logging.Logger for outputting log messages. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, crc32c, and size fields populated. """ start_time = time.time() dst_bucket_url = StorageUrlFromString(dst_url.bucket_url_string) api_selector = gsutil_api.GetApiSelector(provider=dst_url.scheme) encryption_keywrapper = GetEncryptionKeyWrapper(config) encryption_key_sha256 = (encryption_keywrapper.crypto_key_sha256 if encryption_keywrapper else None) # Determine which components, if any, have already been successfully # uploaded. tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.PARALLEL_UPLOAD, api_selector, src_url) (existing_enc_key_sha256, existing_prefix, existing_components) = (ReadParallelUploadTrackerFile( tracker_file_name, logger)) # Ensure that the tracker data is still valid (encryption keys match) and # perform any necessary cleanup. (existing_prefix, existing_components) = ValidateParallelCompositeTrackerData( tracker_file_name, existing_enc_key_sha256, existing_prefix, existing_components, encryption_key_sha256, dst_bucket_url, command_obj, logger, _DeleteTempComponentObjectFn, _RmExceptionHandler) encryption_key_sha256 = (encryption_key_sha256.decode('ascii') if encryption_key_sha256 is not None else None) random_prefix = (existing_prefix if existing_prefix is not None else GenerateComponentObjectPrefix( encryption_key_sha256=encryption_key_sha256)) # Create (or overwrite) the tracker file for the upload. WriteParallelUploadTrackerFile(tracker_file_name, random_prefix, existing_components, encryption_key_sha256=encryption_key_sha256) # Protect the tracker file within calls to Apply. tracker_file_lock = parallelism_framework_util.CreateLock() # Dict to track component info so we may align FileMessage values # before and after the operation. components_info = {} # Get the set of all components that should be uploaded. dst_args = _PartitionFile(canned_acl, dst_obj_metadata.contentType, dst_bucket_url, file_size, fp, random_prefix, src_url, dst_obj_metadata.storageClass, tracker_file_name, tracker_file_lock, encryption_key_sha256=encryption_key_sha256, gzip_encoded=gzip_encoded) (components_to_upload, existing_components, existing_objects_to_delete) = (FilterExistingComponents( dst_args, existing_components, dst_bucket_url, gsutil_api)) # Assign a start message to each different component type for component in components_to_upload: components_info[component.dst_url.url_string] = ( FileMessage.COMPONENT_TO_UPLOAD, component.file_length) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(component.src_url, component.dst_url, time.time(), size=component.file_length, finished=False, component_num=_GetComponentNumber(component.dst_url), message_type=FileMessage.COMPONENT_TO_UPLOAD)) for component in existing_components: component_str = component[0].versionless_url_string components_info[component_str] = (FileMessage.EXISTING_COMPONENT, component[1]) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component[0], time.time(), finished=False, size=component[1], component_num=_GetComponentNumber(component[0]), message_type=FileMessage.EXISTING_COMPONENT)) for component in existing_objects_to_delete: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=False, message_type=FileMessage.EXISTING_OBJECT_TO_DELETE)) # In parallel, copy all of the file parts that haven't already been # uploaded to temporary objects. cp_results = command_obj.Apply( _PerformParallelUploadFileToObject, components_to_upload, copy_exception_handler, ('op_failure_count', 'total_bytes_transferred'), arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE, should_return_results=True) uploaded_components = [] for cp_result in cp_results: uploaded_components.append(cp_result[2]) components = uploaded_components + [i[0] for i in existing_components] if len(components) == len(dst_args): # Only try to compose if all of the components were uploaded successfully. # Sort the components so that they will be composed in the correct order. components = sorted(components, key=_GetComponentNumber) request_components = [] for component_url in components: src_obj_metadata = ( apitools_messages.ComposeRequest.SourceObjectsValueListEntry( name=component_url.object_name)) if component_url.HasGeneration(): src_obj_metadata.generation = long(component_url.generation) request_components.append(src_obj_metadata) composed_object = gsutil_api.ComposeObject( request_components, dst_obj_metadata, preconditions=preconditions, provider=dst_url.scheme, fields=['crc32c', 'generation', 'size'], encryption_tuple=encryption_keywrapper) try: # Make sure only to delete things that we know were successfully # uploaded (as opposed to all of the objects that we attempted to # create) so that we don't delete any preexisting objects, except for # those that were uploaded by a previous, failed run and have since # changed (but still have an old generation lying around). objects_to_delete = components + existing_objects_to_delete command_obj.Apply( _DeleteTempComponentObjectFn, objects_to_delete, _RmExceptionHandler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE) # Assign an end message to each different component type for component in components: component_str = component.versionless_url_string try: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True, component_num=_GetComponentNumber(component), size=components_info[component_str][1], message_type=components_info[component_str][0])) except: # pylint: disable=bare-except PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True)) for component in existing_objects_to_delete: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, component, time.time(), finished=True, message_type=FileMessage.EXISTING_OBJECT_TO_DELETE)) except Exception: # pylint: disable=broad-except # If some of the delete calls fail, don't cause the whole command to # fail. The copy was successful iff the compose call succeeded, so # reduce this to a warning. logger.warn( 'Failed to delete some of the following temporary objects:\n' + '\n'.join(dst_args.keys())) finally: with tracker_file_lock: DeleteTrackerFile(tracker_file_name) else: # Some of the components failed to upload. In this case, we want to exit # without deleting the objects. raise CommandException( 'Some temporary components were not uploaded successfully. ' 'Please retry this upload.') elapsed_time = time.time() - start_time return elapsed_time, composed_object def _ShouldDoParallelCompositeUpload(logger, allow_splitting, src_url, dst_url, file_size, gsutil_api, canned_acl=None): """Determines whether parallel composite upload strategy should be used. Args: logger: for outputting log messages. allow_splitting: If false, then this function returns false. src_url: FileUrl corresponding to a local file. dst_url: CloudUrl corresponding to destination cloud object. file_size: The size of the source file, in bytes. gsutil_api: CloudApi that may be used to check if the destination bucket has any metadata attributes set that would discourage us from using parallel composite uploads. canned_acl: Canned ACL to apply to destination object, if any. Returns: True iff a parallel upload should be performed on the source file. """ global suggested_sliced_transfers, suggested_sliced_transfers_lock parallel_composite_upload_threshold = HumanReadableToBytes( config.get('GSUtil', 'parallel_composite_upload_threshold', DEFAULT_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD)) all_factors_but_size = ( allow_splitting # Don't split the pieces multiple times. and not src_url.IsStream() # We can't partition streams. and not src_url.IsFifo() # We can't partition fifos. and dst_url.scheme == 'gs' # Compose is only for gs. and not canned_acl) # TODO: Implement canned ACL support for compose. # Since parallel composite uploads are disabled by default, make user aware of # them. # TODO: Once compiled crcmod is being distributed by major Linux distributions # remove this check. if (all_factors_but_size and parallel_composite_upload_threshold == 0 and file_size >= PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD): with suggested_sliced_transfers_lock: if not suggested_sliced_transfers.get('suggested'): logger.info('\n'.join( textwrap.wrap( '==> NOTE: You are uploading one or more large file(s), which ' 'would run significantly faster if you enable parallel composite ' 'uploads. This feature can be enabled by editing the ' '"parallel_composite_upload_threshold" value in your .boto ' 'configuration file. However, note that if you do this large files ' 'will be uploaded as ' '`composite objects <https://cloud.google.com/storage/docs/composite-objects>`_,' # pylint: disable=line-too-long 'which means that any user who downloads such objects will need to ' 'have a compiled crcmod installed (see "gsutil help crcmod"). This ' 'is because without a compiled crcmod, computing checksums on ' 'composite objects is so slow that gsutil disables downloads of ' 'composite objects.')) + '\n') suggested_sliced_transfers['suggested'] = True return (all_factors_but_size and parallel_composite_upload_threshold > 0 and file_size >= parallel_composite_upload_threshold) def ExpandUrlToSingleBlr(url_str, gsutil_api, project_id, treat_nonexistent_object_as_subdir=False, logger=None): """Expands wildcard if present in url_str. Args: url_str: String representation of requested url. gsutil_api: gsutil Cloud API instance to use. project_id: project ID to use (for iterators). treat_nonexistent_object_as_subdir: indicates if should treat a non-existent object as a subdir. logger: logging.Logger instance to use for output. If None, the root Logger will be used. Returns: (exp_url, have_existing_dst_container) where exp_url is a StorageUrl and have_existing_dst_container is a bool indicating whether exp_url names an existing directory, bucket, or bucket subdirectory. In the case where we match a subdirectory AND an object, the object is returned. Raises: CommandException: if url_str matched more than 1 URL. """ logger = logger or logging.Logger() # Handle wildcarded url case. if ContainsWildcard(url_str): blr_expansion = list( CreateWildcardIterator(url_str, gsutil_api, project_id=project_id, logger=logger)) if len(blr_expansion) != 1: raise CommandException('Destination (%s) must match exactly 1 URL' % url_str) blr = blr_expansion[0] # BLR is either an OBJECT, PREFIX, or BUCKET; the latter two represent # directories. return (StorageUrlFromString(blr.url_string), not blr.IsObject()) storage_url = StorageUrlFromString(url_str) # Handle non-wildcarded URL. if storage_url.IsFileUrl(): return (storage_url, storage_url.IsDirectory()) # At this point we have a cloud URL. if storage_url.IsBucket(): return (storage_url, True) # For object/prefix URLs, there are four cases that indicate the destination # is a cloud subdirectory; these are always considered to be an existing # container. Checking each case allows gsutil to provide Unix-like # destination folder semantics, but requires up to three HTTP calls, noted # below. # Case 1: If a placeholder object ending with '/' exists. if IsCloudSubdirPlaceholder(storage_url): return (storage_url, True) # Get version of object name without trailing slash for matching prefixes prefix = storage_url.object_name.rstrip('/') # HTTP call to make an eventually consistent check for a matching prefix, # _$folder$, or empty listing. list_iterator = gsutil_api.ListObjects(storage_url.bucket_name, prefix=prefix, delimiter='/', provider=storage_url.scheme, fields=['prefixes', 'items/name']) for obj_or_prefix in list_iterator: # To conserve HTTP calls for the common case, we make a single listing # that covers prefixes and object names. Listing object names covers the # _$folder$ case and the nonexistent-object-as-subdir case. However, if # there are many existing objects for which the target URL is an exact # prefix, this listing could be paginated and span multiple HTTP calls. # If this case becomes common, we could heurestically abort the # listing operation after the first page of results and just query for the # _$folder$ object directly using GetObjectMetadata. # TODO: currently the ListObjects iterator yields objects before prefixes, # because ls depends on this iteration order for proper display. We could # save up to 1ms in determining that a destination is a prefix if we had a # way to yield prefixes first, but this would require poking a major hole # through the abstraction to control this iteration order. if (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.PREFIX and obj_or_prefix.data == prefix + '/'): # Case 2: If there is a matching prefix when listing the destination URL. return (storage_url, True) elif (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT and obj_or_prefix.data.name == storage_url.object_name + '_$folder$'): # Case 3: If a placeholder object matching destination + _$folder$ # exists. return (storage_url, True) elif (obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT and obj_or_prefix.data.name == storage_url.object_name): # The object exists but it is not a container return (storage_url, False) # Case 4: If no objects/prefixes matched, and nonexistent objects should be # treated as subdirectories. return (storage_url, treat_nonexistent_object_as_subdir) def TriggerReauthForDestinationProviderIfNecessary(destination_url, gsutil_api, worker_count): """Makes a request to the destination API provider to trigger reauth. Addresses https://github.com/GoogleCloudPlatform/gsutil/issues/1639. If an API call occurs in a child process, the library that handles reauth will fail. We need to make at least one API call in the main process to allow a user to reauthorize. For cloud source URLs this already happens because the plurality of the source name expansion iterator is checked in the main thread. For cloud destination URLs, only some situations result in a similar API call. In these situations, this function exits without performing an API call. In others, this function performs an API call to trigger reauth. Args: destination_url (StorageUrl): The destination of the transfer. gsutil_api (CloudApiDelegator): API to use for the GetBucket call. worker_count (int): If greater than 1, assume that parallel execution is used. Technically, reauth challenges can be answered in the main process, but they may be triggered multiple times if multithreading is used. Returns: None, but performs an API call if necessary. """ # Only perform this check if the user has opted in. if not config.getbool( 'GSUtil', 'trigger_reauth_challenge_for_parallel_operations', False): return # Reauth is not necessary for non-cloud destinations. if not destination_url.IsCloudUrl(): return # Destination wildcards are expanded by an API call in the main process. if ContainsWildcard(destination_url.url_string): return # If gsutil executes sequentially, all calls will occur in the main process. if worker_count == 1: return try: # The specific API call is not important, but one must occur. gsutil_api.GetBucket( destination_url.bucket_name, fields=['location'], # Single field to limit XML API calls. provider=destination_url.scheme) except Exception as e: if isinstance(e, pyu2f.errors.PluginError): raise # Other exceptions can be ignored. The purpose was just to trigger # a reauth challenge. def FixWindowsNaming(src_url, dst_url): """Translates Windows pathnames to cloud pathnames. Rewrites the destination URL built by ConstructDstUrl(). Args: src_url: Source StorageUrl to be copied. dst_url: The destination StorageUrl built by ConstructDstUrl(). Returns: StorageUrl to use for copy. """ if (src_url.IsFileUrl() and src_url.delim == '\\' and dst_url.IsCloudUrl()): trans_url_str = re.sub(r'\\', '/', dst_url.url_string) dst_url = StorageUrlFromString(trans_url_str) return dst_url def SrcDstSame(src_url, dst_url): """Checks if src_url and dst_url represent the same object or file. We don't handle anything about hard or symbolic links. Args: src_url: Source StorageUrl. dst_url: Destination StorageUrl. Returns: Bool indicator. """ if src_url.IsFileUrl() and dst_url.IsFileUrl(): # Translate a/b/./c to a/b/c, so src=dst comparison below works. new_src_path = os.path.normpath(src_url.object_name) new_dst_path = os.path.normpath(dst_url.object_name) return new_src_path == new_dst_path else: return (src_url.url_string == dst_url.url_string and src_url.generation == dst_url.generation) def _LogCopyOperation(logger, src_url, dst_url, dst_obj_metadata): """Logs copy operation, including Content-Type if appropriate. Args: logger: logger instance to use for output. src_url: Source StorageUrl. dst_url: Destination StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. """ if (dst_url.IsCloudUrl() and dst_obj_metadata and dst_obj_metadata.contentType): content_type_msg = ' [Content-Type=%s]' % dst_obj_metadata.contentType else: content_type_msg = '' if src_url.IsFileUrl() and (src_url.IsStream() or src_url.IsFifo()): src_text = '<STDIN>' if src_url.IsStream() else 'named pipe' logger.info('Copying from %s%s...', src_text, content_type_msg) else: logger.info('Copying %s%s...', src_url.url_string, content_type_msg) # pylint: disable=undefined-variable def _CopyObjToObjInTheCloud(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, decryption_key=None): """Performs copy-in-the cloud from specified src to dest object. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata for source object; must include etag and size. dst_url: Destination CloudUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API instance to use for the copy. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) encryption_keywrapper = GetEncryptionKeyWrapper(config) start_time = time.time() progress_callback = FileProgressCallbackHandler(gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Copying').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call dst_obj = gsutil_api.CopyObject(src_obj_metadata, dst_obj_metadata, src_generation=src_url.generation, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, progress_callback=progress_callback, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, decryption_tuple=decryption_keywrapper, encryption_tuple=encryption_keywrapper) end_time = time.time() result_url = dst_url.Clone() result_url.generation = GenerationFromUrlAndString(result_url, dst_obj.generation) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_CLOUD_COPY, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, src_obj_metadata.size, result_url, dst_obj.md5Hash) def _SetContentTypeFromFile(src_url, dst_obj_metadata): """Detects and sets Content-Type if src_url names a local file. Args: src_url: Source StorageUrl. dst_obj_metadata: Object-specific metadata that should be overidden during the copy. """ # contentType == '' if user requested default type. if (dst_obj_metadata.contentType is None and src_url.IsFileUrl() and not src_url.IsStream() and not src_url.IsFifo()): # Only do content type recognition if src_url is a file. Object-to-object # copies with no -h Content-Type specified re-use the content type of the # source object. object_name = src_url.object_name content_type = None # Streams (denoted by '-') are expected to be 'application/octet-stream' # and 'file' would partially consume them. if object_name != '-': real_file_path = os.path.realpath(object_name) if config.getbool('GSUtil', 'use_magicfile', False) and not IS_WINDOWS: try: p = subprocess.Popen(['file', '-b', '--mime', real_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() p.stdout.close() p.stderr.close() if p.returncode != 0 or error: raise CommandException( 'Encountered error running "file -b --mime %s" ' '(returncode=%d).\n%s' % (real_file_path, p.returncode, error)) # Parse output by removing line delimiter content_type = output.rstrip() content_type = six.ensure_str(content_type) except OSError as e: # 'file' executable may not always be present. raise CommandException( 'Encountered OSError running "file -b --mime %s"\n%s' % (real_file_path, e)) else: _, _, extension = real_file_path.rpartition('.') if extension in COMMON_EXTENSION_RULES: content_type = COMMON_EXTENSION_RULES[extension] else: content_type, _ = mimetypes.guess_type(real_file_path) if not content_type: content_type = DEFAULT_CONTENT_TYPE dst_obj_metadata.contentType = content_type # pylint: disable=undefined-variable def _UploadFileToObjectNonResumable(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, gzip_encoded=False): """Uploads the file using a non-resumable strategy. This function does not support component transfers. Args: src_url: Source StorageUrl to upload. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. """ progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Uploading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call start_time = time.time() encryption_keywrapper = GetEncryptionKeyWrapper(config) if src_url.IsStream() or src_url.IsFifo(): # TODO: gsutil-beta: Provide progress callbacks for streaming uploads. uploaded_object = gsutil_api.UploadObjectStreaming( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_keywrapper, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, gzip_encoded=gzip_encoded) else: uploaded_object = gsutil_api.UploadObject( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, size=src_obj_size, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_keywrapper, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, gzip_encoded=gzip_encoded) end_time = time.time() elapsed_time = end_time - start_time return elapsed_time, uploaded_object # pylint: disable=undefined-variable def _UploadFileToObjectResumable(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, is_component=False, gzip_encoded=False): """Uploads the file using a resumable strategy. Args: src_url: Source FileUrl to upload. Must not be a stream. src_obj_filestream: File pointer to uploadable bytes. src_obj_size (int or None): Size of the source object. dst_url: Destination StorageUrl for the upload. dst_obj_metadata: Metadata for the target object. preconditions: Preconditions for the upload, if any. gsutil_api: gsutil Cloud API instance to use for the upload. logger: for outputting log messages. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Returns: Elapsed upload time, uploaded Object with generation, md5, and size fields populated. """ tracker_file_name = GetTrackerFilePath( dst_url, TrackerFileType.UPLOAD, gsutil_api.GetApiSelector(provider=dst_url.scheme)) encryption_keywrapper = GetEncryptionKeyWrapper(config) encryption_key_sha256 = ( encryption_keywrapper.crypto_key_sha256.decode('ascii') if encryption_keywrapper and encryption_keywrapper.crypto_key_sha256 else None) def _UploadTrackerCallback(serialization_data): """Creates a new tracker file for starting an upload from scratch. This function is called by the gsutil Cloud API implementation and the the serialization data is implementation-specific. Args: serialization_data: Serialization data used in resuming the upload. """ data = { ENCRYPTION_UPLOAD_TRACKER_ENTRY: encryption_key_sha256, SERIALIZATION_UPLOAD_TRACKER_ENTRY: str(serialization_data) } WriteJsonDataToTrackerFile(tracker_file_name, data) # This contains the upload URL, which will uniquely identify the # destination object. tracker_data = GetUploadTrackerData( tracker_file_name, logger, encryption_key_sha256=encryption_key_sha256) if tracker_data: logger.info('Resuming upload for %s', src_url.url_string) retryable = True component_num = _GetComponentNumber(dst_url) if is_component else None progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, component_num=component_num, dst_url=dst_url, operation_name='Uploading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call start_time = time.time() num_startover_attempts = 0 # This loop causes us to retry when the resumable upload failed in a way that # requires starting over with a new upload ID. Retries within a single upload # ID within the current process are handled in # gsutil_api.UploadObjectResumable, and retries within a single upload ID # spanning processes happens if an exception occurs not caught below (which # will leave the tracker file in place, and cause the upload ID to be reused # the next time the user runs gsutil and attempts the same upload). while retryable: try: uploaded_object = gsutil_api.UploadObjectResumable( src_obj_filestream, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, size=src_obj_size, serialization_data=tracker_data, encryption_tuple=encryption_keywrapper, fields=UPLOAD_RETURN_FIELDS, tracker_callback=_UploadTrackerCallback, progress_callback=progress_callback, gzip_encoded=gzip_encoded) retryable = False except ResumableUploadStartOverException as e: logger.info('Caught ResumableUploadStartOverException for upload of %s.' % src_url.url_string) # This can happen, for example, if the server sends a 410 response code. # In that case the current resumable upload ID can't be reused, so delete # the tracker file and try again up to max retries. num_startover_attempts += 1 retryable = (num_startover_attempts < GetNumRetries()) if not retryable: raise # If the server sends a 404 response code, then the upload should only # be restarted if it was the object (and not the bucket) that was missing. try: logger.info('Checking that bucket %s exists before retrying upload...' % dst_obj_metadata.bucket) gsutil_api.GetBucket(dst_obj_metadata.bucket, provider=dst_url.scheme) except AccessDeniedException: # Proceed with deleting the tracker file in the event that the bucket # exists, but the user does not have permission to view its metadata. pass except NotFoundException: raise finally: DeleteTrackerFile(tracker_file_name) logger.info('Deleted tracker file %s for resumable upload of %s before ' 'retrying.' % (tracker_file_name, src_url.url_string)) logger.info( 'Restarting upload of %s from scratch (retry #%d) after exception ' 'indicating we need to start over with a new resumable upload ID: %s' % (src_url.url_string, num_startover_attempts, e)) tracker_data = None src_obj_filestream.seek(0) # Reset the progress callback handler. component_num = _GetComponentNumber(dst_url) if is_component else None progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, component_num=component_num, dst_url=dst_url, operation_name='Uploading').call # Report the retryable error to the global status queue. PutToQueueWithTimeout( gsutil_api.status_queue, RetryableErrorMessage(e, time.time(), num_retries=num_startover_attempts)) time.sleep( min(random.random() * (2*num_startover_attempts), GetMaxRetryDelay())) except ResumableUploadAbortException: retryable = False raise finally: if not retryable: DeleteTrackerFile(tracker_file_name) end_time = time.time() elapsed_time = end_time - start_time return (elapsed_time, uploaded_object) def _SelectUploadCompressionStrategy(object_name, is_component=False, gzip_exts=False, gzip_encoded=False): """Selects how an upload should be compressed. This is a helper function for _UploadFileToObject. Args: object_name: The object name of the source FileUrl. is_component: indicates whether this is a single component or whole file. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: A tuple: (If the file should be gzipped locally, if the file should be gzip transport encoded). """ zipped_file = False gzip_encoded_file = False fname_parts = object_name.split('.') # If gzip_encoded and is_component are marked as true, the file was already # filtered through the original gzip_exts filter and we must compress the # component via gzip transport encoding. if gzip_encoded and is_component: gzip_encoded_file = True elif (gzip_exts == GZIP_ALL_FILES or (gzip_exts and len(fname_parts) > 1 and fname_parts[-1] in gzip_exts)): zipped_file = not gzip_encoded gzip_encoded_file = gzip_encoded return zipped_file, gzip_encoded_file def _ApplyZippedUploadCompression(src_url, src_obj_filestream, src_obj_size, logger): """Compresses a to-be-uploaded local file to save bandwidth. This is a helper function for _UploadFileToObject. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file - will be consumed and closed. src_obj_size (int or None): Size of the source file. logger: for outputting log messages. Returns: StorageUrl path to compressed file, read stream of the compressed file, compressed file size. """ # TODO: Compress using a streaming model as opposed to all at once here. if src_obj_size is not None and src_obj_size >= MIN_SIZE_COMPUTE_LOGGING: logger.info('Compressing %s (to tmp)...', src_url) (gzip_fh, gzip_path) = tempfile.mkstemp() gzip_fp = None try: # Check for temp space. Assume the compressed object is at most 2x # the size of the object (normally should compress to smaller than # the object) if src_url.IsStream() or src_url.IsFifo(): # TODO: Support streaming gzip uploads. # https://github.com/GoogleCloudPlatform/gsutil/issues/364 raise CommandException( 'gzip compression is not currently supported on streaming uploads. ' 'Remove the compression flag or save the streamed output ' 'temporarily to a file before uploading.') if src_obj_size is not None and (CheckFreeSpace(gzip_path) < 2 int(src_obj_size)): raise CommandException('Inadequate temp space available to compress ' '%s. See the CHANGING TEMP DIRECTORIES section ' 'of "gsutil help cp" for more info.' % src_url) compression_level = config.getint('GSUtil', 'gzip_compression_level', DEFAULT_GZIP_COMPRESSION_LEVEL) gzip_fp = gzip.open(gzip_path, 'wb', compresslevel=compression_level) data = src_obj_filestream.read(GZIP_CHUNK_SIZE) while data: gzip_fp.write(data) data = src_obj_filestream.read(GZIP_CHUNK_SIZE) finally: if gzip_fp: gzip_fp.close() os.close(gzip_fh) src_obj_filestream.close() gzip_size = os.path.getsize(gzip_path) compressed_filestream = open(gzip_path, 'rb') return StorageUrlFromString(gzip_path), compressed_filestream, gzip_size def _DelegateUploadFileToObject(upload_delegate, upload_url, upload_stream, zipped_file, gzip_encoded_file, parallel_composite_upload, logger): """Handles setup and tear down logic for uploads. This is a helper function for _UploadFileToObject. Args: upload_delegate: Function that handles uploading the file. upload_url: StorageURL path to the file. upload_stream: Read stream of the file being uploaded. This will be closed after the upload. zipped_file: Flag for if the file is locally compressed prior to calling this function. If true, the local temporary file is deleted after the upload. gzip_encoded_file: Flag for if the file will be uploaded with the gzip transport encoding. If true, a lock is used to limit resource usage. parallel_composite_upload: Set to true if this upload represents a top-level parallel composite upload (not an upload of a component). If true, resource locking is skipped. logger: For outputting log messages. Returns: The elapsed upload time, the uploaded object. """ elapsed_time = None uploaded_object = None try: # Parallel transport compressed uploads use a signifcant amount of memory. # The number of threads that may run concurrently are restricted as a # result. Parallel composite upload's don't actually upload data, but # instead fork for each component and calling _UploadFileToObject # individually. The parallel_composite_upload flag is false for the actual # upload invocation. if gzip_encoded_file and not parallel_composite_upload: with gslib.command.concurrent_compressed_upload_lock: elapsed_time, uploaded_object = upload_delegate() else: elapsed_time, uploaded_object = upload_delegate() finally: # In the zipped_file case, this is the gzip stream. When the gzip stream is # created, the original source stream is closed in # _ApplyZippedUploadCompression. This means that we do not have to # explicitly close the source stream here in the zipped_file case. upload_stream.close() if zipped_file: try: os.unlink(upload_url.object_name) # Windows sometimes complains the temp file is locked when you try to # delete it. except Exception: # pylint: disable=broad-except logger.warning( 'Could not delete %s. This can occur in Windows because the ' 'temporary file is still locked.', upload_url.object_name) return elapsed_time, uploaded_object def _UploadFileToObject(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, command_obj, copy_exception_handler, gzip_exts=None, allow_splitting=True, is_component=False, gzip_encoded=False): """Uploads a local file to an object. Args: src_url: Source FileUrl. src_obj_filestream: Read stream of the source file to be read and closed. src_obj_size (int or None): Size of the source file. dst_url: Destination CloudUrl. dst_obj_metadata: Metadata to be applied to the destination object. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: for outputting log messages. command_obj: command object for use in Apply in parallel composite uploads. copy_exception_handler: For handling copy exceptions during Apply. gzip_exts: List of file extensions to gzip prior to upload, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload. is_component: indicates whether this is a single component or whole file. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts for selecting which files will be encoded. Streaming files compressed is only supported on the JSON GCS API. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ if not dst_obj_metadata or not dst_obj_metadata.contentLanguage: content_language = config.get_value('GSUtil', 'content_language') if content_language: dst_obj_metadata.contentLanguage = content_language upload_url = src_url upload_stream = src_obj_filestream upload_size = src_obj_size zipped_file, gzip_encoded_file = _SelectUploadCompressionStrategy( src_url.object_name, is_component, gzip_exts, gzip_encoded) # The component's parent already printed this debug message. if gzip_encoded_file and not is_component: logger.debug('Using compressed transport encoding for %s.', src_url) elif zipped_file: upload_url, upload_stream, upload_size = _ApplyZippedUploadCompression( src_url, src_obj_filestream, src_obj_size, logger) dst_obj_metadata.contentEncoding = 'gzip' # If we're sending an object with gzip encoding, it's possible it also # has an incompressible content type. Google Cloud Storage will remove # the top layer of compression when serving the object, which would cause # the served content not to match the CRC32C/MD5 hashes stored and make # integrity checking impossible. Therefore we set cache control to # no-transform to ensure it is served in its original form. The caveat is # that to read this object, other clients must then support # accept-encoding:gzip. if not dst_obj_metadata.cacheControl: dst_obj_metadata.cacheControl = 'no-transform' elif 'no-transform' not in dst_obj_metadata.cacheControl.lower(): dst_obj_metadata.cacheControl += ',no-transform' if not is_component: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(upload_url, dst_url, time.time(), message_type=FileMessage.FILE_UPLOAD, size=upload_size, finished=False)) elapsed_time = None uploaded_object = None hash_algs = GetUploadHashAlgs() digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) parallel_composite_upload = _ShouldDoParallelCompositeUpload( logger, allow_splitting, upload_url, dst_url, src_obj_size, gsutil_api, canned_acl=global_copy_helper_opts.canned_acl) non_resumable_upload = ((0 if upload_size is None else upload_size) < ResumableThreshold() or src_url.IsStream() or src_url.IsFifo()) if ((src_url.IsStream() or src_url.IsFifo()) and gsutil_api.GetApiSelector(provider=dst_url.scheme) == ApiSelector.JSON): orig_stream = upload_stream # Add limited seekable properties to the stream via buffering. upload_stream = ResumableStreamingJsonUploadWrapper( orig_stream, GetJsonResumableChunkSize()) if not parallel_composite_upload and len(hash_algs): # Parallel composite uploads calculate hashes per-component in subsequent # calls to this function, but the composition of the final object is a # cloud-only operation. wrapped_filestream = HashingFileUploadWrapper(upload_stream, digesters, hash_algs, upload_url, logger) else: wrapped_filestream = upload_stream def CallParallelCompositeUpload(): return _DoParallelCompositeUpload(upload_stream, upload_url, dst_url, dst_obj_metadata, global_copy_helper_opts.canned_acl, upload_size, preconditions, gsutil_api, command_obj, copy_exception_handler, logger, gzip_encoded=gzip_encoded_file) def CallNonResumableUpload(): return _UploadFileToObjectNonResumable(upload_url, wrapped_filestream, upload_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, gzip_encoded=gzip_encoded_file) def CallResumableUpload(): return _UploadFileToObjectResumable(upload_url, wrapped_filestream, upload_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, is_component=is_component, gzip_encoded=gzip_encoded_file) if parallel_composite_upload: delegate = CallParallelCompositeUpload elif non_resumable_upload: delegate = CallNonResumableUpload else: delegate = CallResumableUpload elapsed_time, uploaded_object = _DelegateUploadFileToObject( delegate, upload_url, upload_stream, zipped_file, gzip_encoded_file, parallel_composite_upload, logger) if not parallel_composite_upload: try: digests = _CreateDigestsFromDigesters(digesters) _CheckHashes(logger, dst_url, uploaded_object, src_url.object_name, digests, is_upload=True) except HashMismatchException: if _RENAME_ON_HASH_MISMATCH: corrupted_obj_metadata = apitools_messages.Object( name=dst_obj_metadata.name, bucket=dst_obj_metadata.bucket, etag=uploaded_object.etag) dst_obj_metadata.name = (dst_url.object_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) gsutil_api.CopyObject(corrupted_obj_metadata, dst_obj_metadata, provider=dst_url.scheme) # If the digest doesn't match, delete the object. gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=uploaded_object.generation, provider=dst_url.scheme) raise result_url = dst_url.Clone() result_url.generation = uploaded_object.generation result_url.generation = GenerationFromUrlAndString(result_url, uploaded_object.generation) if not is_component: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(upload_url, dst_url, time.time(), message_type=FileMessage.FILE_UPLOAD, size=upload_size, finished=True)) return (elapsed_time, uploaded_object.size, result_url, uploaded_object.md5Hash) def _GetDownloadFile(dst_url, src_obj_metadata, logger): """Creates a new download file, and deletes the file that will be replaced. Names and creates a temporary file for this download. Also, if there is an existing file at the path where this file will be placed after the download is completed, that file will be deleted. Args: dst_url: Destination FileUrl. src_obj_metadata: Metadata from the source object. logger: for outputting log messages. Returns: (download_file_name, need_to_unzip) download_file_name: The name of the temporary file to which the object will be downloaded. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. """ dir_name = os.path.dirname(dst_url.object_name) if dir_name and not os.path.exists(dir_name): # Do dir creation in try block so can ignore case where dir already # exists. This is needed to avoid a race condition when running gsutil # -m cp. try: os.makedirs(dir_name) except OSError as e: if e.errno != errno.EEXIST: raise need_to_unzip = False # For gzipped objects download to a temp file and unzip. For the XML API, # this represents the result of a HEAD request. For the JSON API, this is # the stored encoding which the service may not respect. However, if the # server sends decompressed bytes for a file that is stored compressed # (double compressed case), there is no way we can validate the hash and # we will fail our hash check for the object. if ObjectIsGzipEncoded(src_obj_metadata): need_to_unzip = True download_file_name = temporary_file_util.GetTempZipFileName(dst_url) logger.info('Downloading to temp gzip filename %s', download_file_name) else: download_file_name = temporary_file_util.GetTempFileName(dst_url) # If a file exists at the permanent destination (where the file will be moved # after the download is completed), delete it here to reduce disk space # requirements. if os.path.exists(dst_url.object_name): os.unlink(dst_url.object_name) # Downloads open the temporary download file in r+b mode, which requires it # to already exist, so we create it here if it doesn't exist already. if not os.path.exists(download_file_name): fp = open(download_file_name, 'w') fp.close() return download_file_name, need_to_unzip def _ShouldDoSlicedDownload(download_strategy, src_obj_metadata, allow_splitting, logger): """Determines whether the sliced download strategy should be used. Args: download_strategy: CloudApi download strategy. src_obj_metadata: Metadata from the source object. allow_splitting: If false, then this function returns false. logger: logging.Logger for log message output. Returns: True iff a sliced download should be performed on the source file. """ sliced_object_download_threshold = HumanReadableToBytes( config.get('GSUtil', 'sliced_object_download_threshold', DEFAULT_SLICED_OBJECT_DOWNLOAD_THRESHOLD)) max_components = config.getint('GSUtil', 'sliced_object_download_max_components', DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS) # Don't use sliced download if it will prevent us from performing an # integrity check. check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) parallel_hashing = src_obj_metadata.crc32c and UsingCrcmodExtension() hashing_okay = parallel_hashing or check_hashes_config == CHECK_HASH_NEVER use_slice = (allow_splitting and download_strategy is not CloudApi.DownloadStrategy.ONE_SHOT and max_components > 1 and hashing_okay and sliced_object_download_threshold > 0 and src_obj_metadata.size >= sliced_object_download_threshold) if (not use_slice and src_obj_metadata.size >= PARALLEL_COMPOSITE_SUGGESTION_THRESHOLD and not UsingCrcmodExtension() and check_hashes_config != CHECK_HASH_NEVER): with suggested_sliced_transfers_lock: if not suggested_sliced_transfers.get('suggested'): logger.info('\n'.join( textwrap.wrap( '==> NOTE: You are downloading one or more large file(s), which ' 'would run significantly faster if you enabled sliced object ' 'downloads. This feature is enabled by default but requires that ' 'compiled crcmod be installed (see "gsutil help crcmod").')) + '\n') suggested_sliced_transfers['suggested'] = True return use_slice def _PerformSlicedDownloadObjectToFile(cls, args, thread_state=None): """Function argument to Apply for performing sliced downloads. Args: cls: Calling Command class. args: PerformSlicedDownloadObjectToFileArgs tuple describing the target. thread_state: gsutil Cloud API instance to use for the operation. Returns: PerformSlicedDownloadReturnValues named-tuple filled with: component_num: The component number for this download. crc32c: CRC32C hash value (integer) of the downloaded bytes. bytes_transferred: The number of bytes transferred, potentially less than the component size if the download was resumed. component_total_size: The number of bytes corresponding to the whole component size, potentially more than bytes_transferred if the download was resumed. """ gsutil_api = GetCloudApiInstance(cls, thread_state=thread_state) # Deserialize the picklable object metadata. src_obj_metadata = protojson.decode_message(apitools_messages.Object, args.src_obj_metadata_json) hash_algs = GetDownloadHashAlgs(cls.logger, consider_crc32c=src_obj_metadata.crc32c) digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) (bytes_transferred, server_encoding) = (_DownloadObjectToFileResumable( args.src_url, src_obj_metadata, args.dst_url, args.download_file_name, gsutil_api, cls.logger, digesters, component_num=args.component_num, start_byte=args.start_byte, end_byte=args.end_byte, decryption_key=args.decryption_key)) crc32c_val = None if 'crc32c' in digesters: crc32c_val = digesters['crc32c'].crcValue return PerformSlicedDownloadReturnValues(args.component_num, crc32c_val, bytes_transferred, args.end_byte - args.start_byte + 1, server_encoding) def _MaintainSlicedDownloadTrackerFiles(src_obj_metadata, dst_url, download_file_name, logger, api_selector, num_components): """Maintains sliced download tracker files in order to permit resumability. Reads or creates a sliced download tracker file representing this object download. Upon an attempt at cross-process resumption, the contents of the sliced download tracker file are verified to make sure a resumption is possible and appropriate. In the case that a resumption should not be attempted, existing component tracker files are deleted (to prevent child processes from attempting resumption), and a new sliced download tracker file is created. Args: src_obj_metadata: Metadata from the source object. Must include etag and generation. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. logger: for outputting log messages. api_selector: The Cloud API implementation used. num_components: The number of components to perform this download with. """ assert src_obj_metadata.etag tracker_file = None # Only can happen if the resumable threshold is set higher than the # parallel transfer threshold. if src_obj_metadata.size < ResumableThreshold(): return tracker_file_name = GetTrackerFilePath(dst_url, TrackerFileType.SLICED_DOWNLOAD, api_selector) fp = None # Check to see if we should attempt resuming the download. try: fp = open(download_file_name, 'rb') existing_file_size = GetFileSize(fp) # A parallel resumption should be attempted only if the destination file # size is exactly the same as the source size and the tracker file matches. if existing_file_size == src_obj_metadata.size: tracker_file = open(tracker_file_name, 'r') tracker_file_data = json.load(tracker_file) if (tracker_file_data['etag'] == src_obj_metadata.etag and tracker_file_data['generation'] == src_obj_metadata.generation and tracker_file_data['num_components'] == num_components): return else: tracker_file.close() logger.warn('Sliced download tracker file doesn\'t match for ' 'download of %s. Restarting download from scratch.' % dst_url.object_name) except (IOError, ValueError) as e: # Ignore non-existent file (happens first time a download # is attempted on an object), but warn user for other errors. if isinstance(e, ValueError) or e.errno != errno.ENOENT: logger.warn('Couldn\'t read sliced download tracker file (%s): %s. ' 'Restarting download from scratch.' % (tracker_file_name, str(e))) finally: if fp: fp.close() if tracker_file: tracker_file.close() # Delete component tracker files to guarantee download starts from scratch. DeleteDownloadTrackerFiles(dst_url, api_selector) # Create a new sliced download tracker file to represent this download. try: with open(tracker_file_name, 'w') as tracker_file: tracker_file_data = { 'etag': src_obj_metadata.etag, 'generation': src_obj_metadata.generation, 'num_components': num_components, } tracker_file.write(json.dumps(tracker_file_data)) except IOError as e: RaiseUnwritableTrackerFileException(tracker_file_name, e.strerror) class SlicedDownloadFileWrapper(object): """Wraps a file object to be used in GetObjectMedia for sliced downloads. In order to allow resumability, the file object used by each thread in a sliced object download should be wrapped using SlicedDownloadFileWrapper. Passing a SlicedDownloadFileWrapper object to GetObjectMedia will allow the download component tracker file for this component to be updated periodically, while the downloaded bytes are normally written to file. """ def __init__(self, fp, tracker_file_name, src_obj_metadata, start_byte, end_byte): """Initializes the SlicedDownloadFileWrapper. Args: fp: The already-open file object to be used for writing in GetObjectMedia. Data will be written to file starting at the current seek position. tracker_file_name: The name of the tracker file for this component. src_obj_metadata: Metadata from the source object. Must include etag and generation. start_byte: The first byte to be downloaded for this parallel component. end_byte: The last byte to be downloaded for this parallel component. """ self._orig_fp = fp self._tracker_file_name = tracker_file_name self._src_obj_metadata = src_obj_metadata self._last_tracker_file_byte = None self._start_byte = start_byte self._end_byte = end_byte @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def write(self, data): # pylint: disable=invalid-name current_file_pos = self._orig_fp.tell() assert (self._start_byte <= current_file_pos and current_file_pos + len(data) <= self._end_byte + 1) text_util.write_to_fd(self._orig_fp, data) current_file_pos = self._orig_fp.tell() threshold = TRACKERFILE_UPDATE_THRESHOLD if (self._last_tracker_file_byte is None or current_file_pos - self._last_tracker_file_byte > threshold or current_file_pos == self._end_byte + 1): WriteDownloadComponentTrackerFile(self._tracker_file_name, self._src_obj_metadata, current_file_pos) self._last_tracker_file_byte = current_file_pos def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name if whence == os.SEEK_END: self._orig_fp.seek(offset + self._end_byte + 1) else: self._orig_fp.seek(offset, whence) assert self._start_byte <= self._orig_fp.tell() <= self._end_byte + 1 def tell(self): # pylint: disable=invalid-name return self._orig_fp.tell() def flush(self): # pylint: disable=invalid-name self._orig_fp.flush() def close(self): # pylint: disable=invalid-name if self._orig_fp: self._orig_fp.close() def _PartitionObject(src_url, src_obj_metadata, dst_url, download_file_name, decryption_key=None): """Partitions an object into components to be downloaded. Each component is a byte range of the object. The byte ranges of the returned components are mutually exclusive and collectively exhaustive. The byte ranges are inclusive at both end points. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object with non-pickleable fields removed. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for the download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: components_to_download: A list of PerformSlicedDownloadObjectToFileArgs to be used in Apply for the sliced download. """ sliced_download_component_size = HumanReadableToBytes( config.get('GSUtil', 'sliced_object_download_component_size', DEFAULT_SLICED_OBJECT_DOWNLOAD_COMPONENT_SIZE)) max_components = config.getint('GSUtil', 'sliced_object_download_max_components', DEFAULT_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS) num_components, component_size = _GetPartitionInfo( src_obj_metadata.size, max_components, sliced_download_component_size) components_to_download = [] component_lengths = [] for i in range(num_components): start_byte = i * component_size end_byte = min((i + 1) * (component_size) - 1, src_obj_metadata.size - 1) component_lengths.append(end_byte - start_byte + 1) # We need to serialize src_obj_metadata for pickling since it can # contain nested classes such as custom metadata. src_obj_metadata_json = protojson.encode_message(src_obj_metadata) components_to_download.append( PerformSlicedDownloadObjectToFileArgs(i, src_url, src_obj_metadata_json, dst_url, download_file_name, start_byte, end_byte, decryption_key)) return components_to_download, component_lengths def _DoSlicedDownload(src_url, src_obj_metadata, dst_url, download_file_name, command_obj, logger, copy_exception_handler, api_selector, decryption_key=None, status_queue=None): """Downloads a cloud object to a local file using sliced download. Byte ranges are decided for each thread/process, and then the parts are downloaded in parallel. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. command_obj: command object for use in Apply in parallel composite uploads. logger: for outputting log messages. copy_exception_handler: For handling copy exceptions during Apply. api_selector: The Cloud API implementation used. decryption_key: Base64-encoded decryption key for the source object, if any. status_queue: Queue for posting file messages for UI/Analytics. Returns: (bytes_transferred, crc32c) bytes_transferred: Number of bytes transferred from server this call. crc32c: a crc32c hash value (integer) for the downloaded bytes, or None if crc32c hashing wasn't performed. """ # CustomerEncryptionValue is a subclass and thus not pickleable for # multiprocessing, but at this point we already have the matching key, # so just discard the metadata. src_obj_metadata.customerEncryption = None components_to_download, component_lengths = _PartitionObject( src_url, src_obj_metadata, dst_url, download_file_name, decryption_key) num_components = len(components_to_download) _MaintainSlicedDownloadTrackerFiles(src_obj_metadata, dst_url, download_file_name, logger, api_selector, num_components) # Resize the download file so each child process can seek to its start byte. with open(download_file_name, 'r+b') as fp: fp.truncate(src_obj_metadata.size) # Assign a start FileMessage to each component for (i, component) in enumerate(components_to_download): size = component.end_byte - component.start_byte + 1 download_start_byte = GetDownloadStartByte(src_obj_metadata, dst_url, api_selector, component.start_byte, size, i) bytes_already_downloaded = download_start_byte - component.start_byte PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, time.time(), size=size, finished=False, component_num=i, message_type=FileMessage.COMPONENT_TO_DOWNLOAD, bytes_already_downloaded=bytes_already_downloaded)) cp_results = command_obj.Apply( _PerformSlicedDownloadObjectToFile, components_to_download, copy_exception_handler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SLICE, should_return_results=True) if len(cp_results) < num_components: raise CommandException( 'Some components of %s were not downloaded successfully. ' 'Please retry this download.' % dst_url.object_name) # Crc32c hashes have to be concatenated in the correct order. cp_results = sorted(cp_results, key=attrgetter('component_num')) crc32c = cp_results[0].crc32c if crc32c is not None: for i in range(1, num_components): crc32c = ConcatCrc32c(crc32c, cp_results[i].crc32c, component_lengths[i]) bytes_transferred = 0 expect_gzip = ObjectIsGzipEncoded(src_obj_metadata) # Assign an end FileMessage to each component for cp_result in cp_results: PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, time.time(), size=cp_result.component_total_size, finished=True, component_num=cp_result.component_num, message_type=FileMessage.COMPONENT_TO_DOWNLOAD)) bytes_transferred += cp_result.bytes_transferred server_gzip = (cp_result.server_encoding and cp_result.server_encoding.lower().endswith('gzip')) # If the server gzipped any components on the fly, we will have no chance of # properly reconstructing the file. if server_gzip and not expect_gzip: raise CommandException( 'Download of %s failed because the server sent back data with an ' 'unexpected encoding.' % dst_url.object_name) return bytes_transferred, crc32c def _DownloadObjectToFileResumable(src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, logger, digesters, component_num=None, start_byte=0, end_byte=None, decryption_key=None): """Downloads an object to a local file using the resumable strategy. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. digesters: Digesters corresponding to the hash algorithms that will be used for validation. component_num: Which component of a sliced download this call is for, or None if this is not a sliced download. start_byte: The first byte of a byte range for a sliced download. end_byte: The last byte of a byte range for a sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. """ if end_byte is None: end_byte = src_obj_metadata.size - 1 download_size = end_byte - start_byte + 1 is_sliced = component_num is not None api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) server_encoding = None # Used for logging download_name = dst_url.object_name if is_sliced: download_name += ' component %d' % component_num fp = None try: fp = open(download_file_name, 'r+b') fp.seek(start_byte) api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) existing_file_size = GetFileSize(fp) tracker_file_name, download_start_byte = ReadOrCreateDownloadTrackerFile( src_obj_metadata, dst_url, logger, api_selector, start_byte, existing_file_size, component_num, ) if download_start_byte < start_byte or download_start_byte > end_byte + 1: DeleteTrackerFile(tracker_file_name) raise CommandException( 'Resumable download start point for %s is not in the correct byte ' 'range. Deleting tracker file, so if you re-try this download it ' 'will start from scratch' % download_name) download_complete = (download_start_byte == start_byte + download_size) resuming = (download_start_byte != start_byte) and not download_complete if resuming: logger.info('Resuming download for %s', download_name) elif download_complete: logger.info( 'Download already complete for %s, skipping download but ' 'will run integrity checks.', download_name) # This is used for resuming downloads, but also for passing the mediaLink # and size into the download for new downloads so that we can avoid # making an extra HTTP call. serialization_data = GetDownloadSerializationData( src_obj_metadata, progress=download_start_byte, user_project=gsutil_api.user_project) if resuming or download_complete: # Catch up our digester with the hash data. bytes_digested = 0 total_bytes_to_digest = download_start_byte - start_byte hash_callback = ProgressCallbackWithTimeout( total_bytes_to_digest, FileProgressCallbackHandler(gsutil_api.status_queue, component_num=component_num, src_url=src_url, dst_url=dst_url, operation_name='Hashing').call) while bytes_digested < total_bytes_to_digest: bytes_to_read = min(DEFAULT_FILE_BUFFER_SIZE, total_bytes_to_digest - bytes_digested) data = fp.read(bytes_to_read) bytes_digested += bytes_to_read for alg_name in digesters: digesters[alg_name].update(six.ensure_binary(data)) hash_callback.Progress(len(data)) elif not is_sliced: # Delete file contents and start entire object download from scratch. fp.truncate(0) existing_file_size = 0 progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, start_byte=start_byte, override_total_size=download_size, src_url=src_url, dst_url=dst_url, component_num=component_num, operation_name='Downloading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call if is_sliced and src_obj_metadata.size >= ResumableThreshold(): fp = SlicedDownloadFileWrapper(fp, tracker_file_name, src_obj_metadata, start_byte, end_byte) compressed_encoding = ObjectIsGzipEncoded(src_obj_metadata) # TODO: With gzip encoding (which may occur on-the-fly and not be part of # the object's metadata), when we request a range to resume, it's possible # that the server will just resend the entire object, which means our # caught-up hash will be incorrect. We recalculate the hash on # the local file in the case of a failed gzip hash anyway, but it would # be better if we actively detected this case. if not download_complete: fp.seek(download_start_byte) server_encoding = gsutil_api.GetObjectMedia( src_url.bucket_name, src_url.object_name, fp, start_byte=download_start_byte, end_byte=end_byte, compressed_encoding=compressed_encoding, generation=src_url.generation, object_size=src_obj_metadata.size, download_strategy=CloudApi.DownloadStrategy.RESUMABLE, provider=src_url.scheme, serialization_data=serialization_data, digesters=digesters, progress_callback=progress_callback, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) except ResumableDownloadException as e: logger.warning('Caught ResumableDownloadException (%s) for download of %s.', e.reason, download_name) raise finally: if fp: fp.close() bytes_transferred = end_byte - download_start_byte + 1 return bytes_transferred, server_encoding def _DownloadObjectToFileNonResumable(src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, digesters, decryption_key=None): """Downloads an object to a local file using the non-resumable strategy. This function does not support component transfers. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. download_file_name: Temporary file name to be used for download. gsutil_api: gsutil Cloud API instance to use for the download. digesters: Digesters corresponding to the hash algorithms that will be used for validation. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (bytes_transferred, server_encoding) bytes_transferred: Number of bytes transferred from server this call. server_encoding: Content-encoding string if it was detected that the server sent encoded bytes during transfer, None otherwise. """ fp = None try: fp = open(download_file_name, 'w') # This is used to pass the mediaLink and the size into the download so that # we can avoid making an extra HTTP call. serialization_data = GetDownloadSerializationData( src_obj_metadata, 0, user_project=gsutil_api.user_project) progress_callback = FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Downloading').call if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call server_encoding = gsutil_api.GetObjectMedia( src_url.bucket_name, src_url.object_name, fp, generation=src_url.generation, object_size=src_obj_metadata.size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=src_url.scheme, serialization_data=serialization_data, digesters=digesters, progress_callback=progress_callback, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) finally: if fp: fp.close() return src_obj_metadata.size, server_encoding def _DownloadObjectToFile(src_url, src_obj_metadata, dst_url, gsutil_api, logger, command_obj, copy_exception_handler, allow_splitting=True, decryption_key=None, is_rsync=False, preserve_posix=False, use_stet=False): """Downloads an object to a local file. Args: src_url: Source CloudUrl. src_obj_metadata: Metadata from the source object. dst_url: Destination FileUrl. gsutil_api: gsutil Cloud API instance to use for the download. logger: for outputting log messages. command_obj: command object for use in Apply in sliced downloads. copy_exception_handler: For handling copy exceptions during Apply. allow_splitting: Whether or not to allow sliced download. decryption_key: Base64-encoded decryption key for the source object, if any. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve POSIX attributes. use_stet: Decrypt downloaded file with STET binary if available on system. Returns: (elapsed_time, bytes_transferred, dst_url, md5), where time elapsed excludes initial GET. Raises: FileConcurrencySkipError: if this download is already in progress. CommandException: if other errors encountered. """ global open_files_map, open_files_lock if dst_url.object_name.endswith(dst_url.delim): logger.warn('\n'.join( textwrap.wrap( 'Skipping attempt to download to filename ending with slash (%s). This ' 'typically happens when using gsutil to download from a subdirectory ' 'created by the Cloud Console (https://cloud.google.com/console)' % dst_url.object_name))) # The warning above is needed because errors might get ignored # for parallel processing. raise InvalidUrlError('Invalid destination path: %s' % dst_url.object_name) api_selector = gsutil_api.GetApiSelector(provider=src_url.scheme) download_strategy = _SelectDownloadStrategy(dst_url) sliced_download = _ShouldDoSlicedDownload(download_strategy, src_obj_metadata, allow_splitting, logger) download_file_name, need_to_unzip = _GetDownloadFile(dst_url, src_obj_metadata, logger) # Ensure another process/thread is not already writing to this file. with open_files_lock: if open_files_map.get(download_file_name, False): raise FileConcurrencySkipError open_files_map[download_file_name] = True # Set up hash digesters. consider_md5 = src_obj_metadata.md5Hash and not sliced_download hash_algs = GetDownloadHashAlgs(logger, consider_md5=consider_md5, consider_crc32c=src_obj_metadata.crc32c) digesters = dict((alg, hash_algs[alg]()) for alg in hash_algs or {}) # Tracks whether the server used a gzip encoding. server_encoding = None download_complete = (src_obj_metadata.size == 0) bytes_transferred = 0 start_time = time.time() if not download_complete: if sliced_download: (bytes_transferred, crc32c) = (_DoSlicedDownload(src_url, src_obj_metadata, dst_url, download_file_name, command_obj, logger, copy_exception_handler, api_selector, decryption_key=decryption_key, status_queue=gsutil_api.status_queue)) if 'crc32c' in digesters: digesters['crc32c'].crcValue = crc32c elif download_strategy is CloudApi.DownloadStrategy.ONE_SHOT: bytes_transferred, server_encoding = _DownloadObjectToFileNonResumable( src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, digesters, decryption_key=decryption_key, ) elif download_strategy is CloudApi.DownloadStrategy.RESUMABLE: bytes_transferred, server_encoding = _DownloadObjectToFileResumable( src_url, src_obj_metadata, dst_url, download_file_name, gsutil_api, logger, digesters, decryption_key=decryption_key, ) else: raise CommandException('Invalid download strategy %s chosen for' 'file %s' % (download_strategy, download_file_name)) end_time = time.time() server_gzip = server_encoding and server_encoding.lower().endswith('gzip') local_md5 = _ValidateAndCompleteDownload(logger, src_url, src_obj_metadata, dst_url, need_to_unzip, server_gzip, digesters, hash_algs, download_file_name, api_selector, bytes_transferred, gsutil_api, is_rsync=is_rsync, preserve_posix=preserve_posix, use_stet=use_stet) with open_files_lock: open_files_map.delete(download_file_name) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, message_time=end_time, message_type=FileMessage.FILE_DOWNLOAD, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, bytes_transferred, dst_url, local_md5) def _ValidateAndCompleteDownload(logger, src_url, src_obj_metadata, dst_url, need_to_unzip, server_gzip, digesters, hash_algs, temporary_file_name, api_selector, bytes_transferred, gsutil_api, is_rsync=False, preserve_posix=False, use_stet=False): """Validates and performs necessary operations on a downloaded file. Validates the integrity of the downloaded file using hash_algs. If the file was compressed (temporarily), the file will be decompressed. Then, if the integrity of the file was successfully validated, the file will be moved from its temporary download location to its permanent location on disk. Args: logger: For outputting log messages. src_url: StorageUrl for the source object. src_obj_metadata: Metadata for the source object, potentially containing hash values. dst_url: StorageUrl describing the destination file. need_to_unzip: If true, a temporary zip file was used and must be uncompressed as part of validation. server_gzip: If true, the server gzipped the bytes (regardless of whether the object metadata claimed it was gzipped). digesters: dict of {string, hash digester} that contains up-to-date digests computed during the download. If a digester for a particular algorithm is None, an up-to-date digest is not available and the hash must be recomputed from the local file. hash_algs: dict of {string, hash algorithm} that can be used if digesters don't have up-to-date digests. temporary_file_name: Temporary file name that was used for download. api_selector: The Cloud API implementation used (used tracker file naming). bytes_transferred: Number of bytes downloaded (used for logging). gsutil_api: Cloud API to use for service and status. is_rsync: Whether or not the caller is the rsync function. Used to determine if timeCreated should be used. preserve_posix: Whether or not to preserve the posix attributes. use_stet: If True, attempt to decrypt downloaded files with the STET binary if it's present on the system. Returns: An MD5 of the local file, if one was calculated as part of the integrity check. """ final_file_name = dst_url.object_name digesters_succeeded = True for alg in digesters: # If we get a digester with a None algorithm, the underlying # implementation failed to calculate a digest, so we will need to # calculate one from scratch. if not digesters[alg]: digesters_succeeded = False break if digesters_succeeded: local_hashes = _CreateDigestsFromDigesters(digesters) else: local_hashes = _CreateDigestsFromLocalFile(gsutil_api.status_queue, hash_algs, temporary_file_name, src_url, src_obj_metadata) digest_verified = True hash_invalid_exception = None try: _CheckHashes(logger, src_url, src_obj_metadata, final_file_name, local_hashes) DeleteDownloadTrackerFiles(dst_url, api_selector) except HashMismatchException as e: # If an non-gzipped object gets sent with gzip content encoding, the hash # we calculate will match the gzipped bytes, not the original object. Thus, # we'll need to calculate and check it after unzipping. if server_gzip: logger.debug('Hash did not match but server gzipped the content, will ' 'recalculate.') digest_verified = False elif api_selector == ApiSelector.XML: logger.debug( 'Hash did not match but server may have gzipped the content, will ' 'recalculate.') # Save off the exception in case this isn't a gzipped file. hash_invalid_exception = e digest_verified = False else: DeleteDownloadTrackerFiles(dst_url, api_selector) if _RENAME_ON_HASH_MISMATCH: os.rename(temporary_file_name, final_file_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) else: os.unlink(temporary_file_name) raise if not (need_to_unzip or server_gzip): unzipped_temporary_file_name = temporary_file_name else: # This will not result in the same string as temporary_file_name b/c # GetTempFileName returns ".gstmp" and gzipped temp files have ".gztmp". unzipped_temporary_file_name = temporary_file_util.GetTempFileName(dst_url) # Log that we're uncompressing if the file is big enough that # decompressing would make it look like the transfer "stalled" at the end. if bytes_transferred > TEN_MIB: logger.info('Uncompressing temporarily gzipped file to %s...', final_file_name) gzip_fp = None try: # Downloaded temporarily gzipped file, unzip to file without '_.gztmp' # suffix. gzip_fp = gzip.open(temporary_file_name, 'rb') with open(unzipped_temporary_file_name, 'wb') as f_out: data = gzip_fp.read(GZIP_CHUNK_SIZE) while data: f_out.write(data) data = gzip_fp.read(GZIP_CHUNK_SIZE) except IOError as e: # In the XML case where we don't know if the file was gzipped, raise # the original hash exception if we find that it wasn't. if 'Not a gzipped file' in str(e) and hash_invalid_exception: # Linter improperly thinks we're raising None despite the above check. # pylint: disable=raising-bad-type raise hash_invalid_exception finally: if gzip_fp: gzip_fp.close() os.unlink(temporary_file_name) if not digest_verified: try: # Recalculate hashes on the unzipped local file. local_hashes = _CreateDigestsFromLocalFile(gsutil_api.status_queue, hash_algs, unzipped_temporary_file_name, src_url, src_obj_metadata) _CheckHashes(logger, src_url, src_obj_metadata, final_file_name, local_hashes) DeleteDownloadTrackerFiles(dst_url, api_selector) except HashMismatchException: DeleteDownloadTrackerFiles(dst_url, api_selector) if _RENAME_ON_HASH_MISMATCH: os.rename( unzipped_temporary_file_name, unzipped_temporary_file_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) else: os.unlink(unzipped_temporary_file_name) raise if use_stet: # Decrypt data using STET binary. stet_util.decrypt_download(src_url, dst_url, unzipped_temporary_file_name, logger) os.rename(unzipped_temporary_file_name, final_file_name) ParseAndSetPOSIXAttributes(final_file_name, src_obj_metadata, is_rsync=is_rsync, preserve_posix=preserve_posix) if 'md5' in local_hashes: return local_hashes['md5'] def _CopyFileToFile(src_url, dst_url, status_queue=None, src_obj_metadata=None): """Copies a local file to a local file. Args: src_url: Source FileUrl. dst_url: Destination FileUrl. status_queue: Queue for posting file messages for UI/Analytics. src_obj_metadata: An apitools Object that may contain file size, or None. Returns: (elapsed_time, bytes_transferred, dst_url, md5=None). Raises: CommandException: if errors encountered. """ src_fp = GetStreamFromFileUrl(src_url) dir_name = os.path.dirname(dst_url.object_name) if dir_name: try: os.makedirs(dir_name) except OSError as e: if e.errno != errno.EEXIST: raise with open(dst_url.object_name, 'wb') as dst_fp: start_time = time.time() shutil.copyfileobj(src_fp, dst_fp) if not src_url.IsStream(): src_fp.close() # Explicitly close the src fp - necessary if it is a fifo. end_time = time.time() PutToQueueWithTimeout( status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_LOCAL_COPY, size=src_obj_metadata.size if src_obj_metadata else None, finished=True)) return (end_time - start_time, os.path.getsize(dst_url.object_name), dst_url, None) def _DummyTrackerCallback(_): pass # pylint: disable=undefined-variable def _CopyObjToObjDaisyChainMode(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, decryption_key=None): """Copies from src_url to dst_url in "daisy chain" mode. See -D OPTION documentation about what daisy chain mode is. Args: src_url: Source CloudUrl src_obj_metadata: Metadata from source object dst_url: Destination CloudUrl dst_obj_metadata: Object-specific metadata that should be overidden during the copy. preconditions: Preconditions to use for the copy. gsutil_api: gsutil Cloud API to use for the copy. logger: For outputting log messages. decryption_key: Base64-encoded decryption key for the source object, if any. Returns: (elapsed_time, bytes_transferred, dst_url with generation, md5 hash of destination) excluding overhead like initial GET. Raises: CommandException: if errors encountered. """ # We don't attempt to preserve ACLs across providers because # GCS and S3 support different ACLs and disjoint principals. if (global_copy_helper_opts.preserve_acl and src_url.scheme != dst_url.scheme): raise NotImplementedError('Cross-provider cp -p not supported') if not global_copy_helper_opts.preserve_acl: dst_obj_metadata.acl = [] # Don't use callbacks for downloads on the daisy chain wrapper because # upload callbacks will output progress, but respect test hooks if present. progress_callback = None if global_copy_helper_opts.test_callback_file: with open(global_copy_helper_opts.test_callback_file, 'rb') as test_fp: progress_callback = pickle.loads(test_fp.read()).call compressed_encoding = ObjectIsGzipEncoded(src_obj_metadata) encryption_keywrapper = GetEncryptionKeyWrapper(config) start_time = time.time() upload_fp = DaisyChainWrapper(src_url, src_obj_metadata.size, gsutil_api, compressed_encoding=compressed_encoding, progress_callback=progress_callback, decryption_key=decryption_key) uploaded_object = None if src_obj_metadata.size == 0: # Resumable uploads of size 0 are not supported. uploaded_object = gsutil_api.UploadObject( upload_fp, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, size=src_obj_metadata.size, encryption_tuple=encryption_keywrapper) else: # TODO: Support process-break resumes. This will resume across connection # breaks and server errors, but the tracker callback is a no-op so this # won't resume across gsutil runs. # TODO: Test retries via test_callback_file. uploaded_object = gsutil_api.UploadObjectResumable( upload_fp, object_metadata=dst_obj_metadata, canned_acl=global_copy_helper_opts.canned_acl, preconditions=preconditions, provider=dst_url.scheme, fields=UPLOAD_RETURN_FIELDS, size=src_obj_metadata.size, progress_callback=FileProgressCallbackHandler( gsutil_api.status_queue, src_url=src_url, dst_url=dst_url, operation_name='Uploading').call, tracker_callback=_DummyTrackerCallback, encryption_tuple=encryption_keywrapper) end_time = time.time() try: _CheckCloudHashes(logger, src_url, dst_url, src_obj_metadata, uploaded_object) except HashMismatchException: if _RENAME_ON_HASH_MISMATCH: corrupted_obj_metadata = apitools_messages.Object( name=dst_obj_metadata.name, bucket=dst_obj_metadata.bucket, etag=uploaded_object.etag) dst_obj_metadata.name = (dst_url.object_name + _RENAME_ON_HASH_MISMATCH_SUFFIX) decryption_keywrapper = CryptoKeyWrapperFromKey(decryption_key) gsutil_api.CopyObject(corrupted_obj_metadata, dst_obj_metadata, provider=dst_url.scheme, decryption_tuple=decryption_keywrapper, encryption_tuple=encryption_keywrapper) # If the digest doesn't match, delete the object. gsutil_api.DeleteObject(dst_url.bucket_name, dst_url.object_name, generation=uploaded_object.generation, provider=dst_url.scheme) raise result_url = dst_url.Clone() result_url.generation = GenerationFromUrlAndString(result_url, uploaded_object.generation) PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, end_time, message_type=FileMessage.FILE_DAISY_COPY, size=src_obj_metadata.size, finished=True)) return (end_time - start_time, src_obj_metadata.size, result_url, uploaded_object.md5Hash) def GetSourceFieldsNeededForCopy(dst_is_cloud, skip_unsupported_objects, preserve_acl, is_rsync=False, preserve_posix=False, delete_source=False, file_size_will_change=False): """Determines the metadata fields needed for a copy operation. This function returns the fields we will need to successfully copy any cloud objects that might be iterated. By determining this prior to iteration, the cp command can request this metadata directly from the iterator's get/list calls, avoiding the need for a separate get metadata HTTP call for each iterated result. As a trade-off, filtering objects at the leaf nodes of the iteration (based on a remaining wildcard) is more expensive. This is because more metadata will be requested when object name is all that is required for filtering. The rsync command favors fast listing and comparison, and makes the opposite trade-off, optimizing for the low-delta case by making per-object get metadata HTTP call so that listing can return minimal metadata. It uses this function to determine what is needed for get metadata HTTP calls. Args: dst_is_cloud: if true, destination is a Cloud URL. skip_unsupported_objects: if true, get metadata for skipping unsupported object types. preserve_acl: if true, get object ACL. is_rsync: if true, the calling function is rsync. Determines if metadata is needed to verify download. preserve_posix: if true, retrieves POSIX attributes into user metadata. delete_source: if true, source object will be deleted after the copy (mv command). file_size_will_change: if true, do not try to record file size. Returns: List of necessary field metadata field names. """ src_obj_fields_set = set() if dst_is_cloud: # For cloud or daisy chain copy, we need every copyable field. # If we're not modifying or overriding any of the fields, we can get # away without retrieving the object metadata because the copy # operation can succeed with just the destination bucket and object # name. But if we are sending any metadata, the JSON API will expect a # complete object resource. Since we want metadata like the object size for # our own tracking, we just get all of the metadata here. src_obj_fields_set.update([ 'cacheControl', 'componentCount', 'contentDisposition', 'contentEncoding', 'contentLanguage', 'contentType', 'crc32c', 'customerEncryption', 'etag', 'generation', 'md5Hash', 'mediaLink', 'metadata', 'metageneration', 'storageClass', 'timeCreated', ]) # We only need the ACL if we're going to preserve it. if preserve_acl: src_obj_fields_set.update(['acl']) if not file_size_will_change: src_obj_fields_set.update(['size']) else: # Just get the fields needed to perform and validate the download. src_obj_fields_set.update([ 'crc32c', 'contentEncoding', 'contentType', 'customerEncryption', 'etag', 'mediaLink', 'md5Hash', 'size', 'generation', ]) if is_rsync: src_obj_fields_set.update(['metadata/%s' % MTIME_ATTR, 'timeCreated']) if preserve_posix: posix_fields = [ 'metadata/%s' % ATIME_ATTR, 'metadata/%s' % MTIME_ATTR, 'metadata/%s' % GID_ATTR, 'metadata/%s' % MODE_ATTR, 'metadata/%s' % UID_ATTR, ] src_obj_fields_set.update(posix_fields) if delete_source: src_obj_fields_set.update([ 'storageClass', 'timeCreated', ]) if skip_unsupported_objects: src_obj_fields_set.update(['storageClass']) return list(src_obj_fields_set) # Map of (lowercase) storage classes with early deletion charges to their # minimum lifetime in seconds. EARLY_DELETION_MINIMUM_LIFETIME = { 'nearline': 30 * SECONDS_PER_DAY, 'coldline': 90 * SECONDS_PER_DAY, 'archive': 365 * SECONDS_PER_DAY } def WarnIfMvEarlyDeletionChargeApplies(src_url, src_obj_metadata, logger): """Warns when deleting a gs:// object could incur an early deletion charge. This function inspects metadata for Google Cloud Storage objects that are subject to early deletion charges (such as Nearline), and warns when performing operations like mv that would delete them. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with necessary fields (per GetSourceFieldsNeededForCopy). logger: logging.Logger for outputting warning. """ if (src_url.scheme == 'gs' and src_obj_metadata and src_obj_metadata.timeCreated and src_obj_metadata.storageClass): object_storage_class = src_obj_metadata.storageClass.lower() early_deletion_cutoff_seconds = EARLY_DELETION_MINIMUM_LIFETIME.get( object_storage_class, None) if early_deletion_cutoff_seconds: minimum_delete_age = ( early_deletion_cutoff_seconds + ConvertDatetimeToPOSIX(src_obj_metadata.timeCreated)) if time.time() < minimum_delete_age: logger.warn( 'Warning: moving %s object %s may incur an early deletion ' 'charge, because the original object is less than %s ' 'days old according to the local system time.', object_storage_class, src_url.url_string, early_deletion_cutoff_seconds // SECONDS_PER_DAY) def MaybeSkipUnsupportedObject(src_url, src_obj_metadata): """Skips unsupported object types if requested. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with storageClass field (per GetSourceFieldsNeededForCopy). Raises: SkipGlacierError: if skipping a s3 Glacier object. """ if (src_url.scheme == 's3' and global_copy_helper_opts.skip_unsupported_objects and src_obj_metadata.storageClass == 'GLACIER'): raise SkipGlacierError() def GetDecryptionCSEK(src_url, src_obj_metadata): """Ensures a matching decryption key is available for the source object. Args: src_url: CloudUrl for the source object. src_obj_metadata: source object metadata with optional customerEncryption field. Raises: EncryptionException if the object is encrypted and no matching key is found. Returns: Base64-encoded decryption key string if the object is encrypted and a matching key is found, or None if object is not encrypted. """ if src_obj_metadata.customerEncryption: decryption_key = FindMatchingCSEKInBotoConfig( src_obj_metadata.customerEncryption.keySha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s' % (src_obj_metadata.customerEncryption.keySha256, src_url)) return decryption_key # pylint: disable=undefined-variable # pylint: disable=too-many-statements def PerformCopy(logger, src_url, dst_url, gsutil_api, command_obj, copy_exception_handler, src_obj_metadata=None, allow_splitting=True, headers=None, manifest=None, gzip_exts=None, is_rsync=False, preserve_posix=False, gzip_encoded=False, use_stet=False): """Performs copy from src_url to dst_url, handling various special cases. Args: logger: for outputting log messages. src_url: Source StorageUrl. dst_url: Destination StorageUrl. gsutil_api: gsutil Cloud API instance to use for the copy. command_obj: command object for use in Apply in parallel composite uploads and sliced object downloads. copy_exception_handler: for handling copy exceptions during Apply. src_obj_metadata: If source URL is a cloud object, source object metadata with all necessary fields (per GetSourceFieldsNeededForCopy). Required for cloud source URLs. If source URL is a file, an apitools Object that may contain file size, or None. allow_splitting: Whether to allow the file to be split into component pieces for an parallel composite upload or download. headers: optional headers to use for the copy operation. manifest: optional manifest for tracking copy operations. gzip_exts: List of file extensions to gzip, if any. If gzip_exts is GZIP_ALL_FILES, gzip all files. is_rsync: Whether or not the caller is the rsync command. preserve_posix: Whether or not to preserve posix attributes. gzip_encoded: Whether to use gzip transport encoding for the upload. Used in conjunction with gzip_exts. Streaming files compressed is only supported on the JSON GCS API. use_stet: If True, will perform STET encryption or decryption using the binary specified in the boto config or PATH. Returns: (elapsed_time, bytes_transferred, version-specific dst_url) excluding overhead like initial GET. Raises: ItemExistsError: if no clobber flag is specified and the destination object already exists. SkipUnsupportedObjectError: if skip_unsupported_objects flag is specified and the source is an unsupported type. CommandException: if other errors encountered. """ # TODO: Remove elapsed_time as it is currently unused by all callers. if headers: dst_obj_headers = headers.copy() else: dst_obj_headers = {} # Create a metadata instance for each destination object so metadata # such as content-type can be applied per-object. # Initialize metadata from any headers passed in via -h. dst_obj_metadata = ObjectMetadataFromHeaders(dst_obj_headers) if dst_url.IsCloudUrl() and dst_url.scheme == 'gs': preconditions = PreconditionsFromHeaders(dst_obj_headers) else: preconditions = Preconditions() src_obj_filestream = None decryption_key = None copy_in_the_cloud = False if src_url.IsCloudUrl(): if (dst_url.IsCloudUrl() and src_url.scheme == dst_url.scheme and not global_copy_helper_opts.daisy_chain): copy_in_the_cloud = True if global_copy_helper_opts.perform_mv: WarnIfMvEarlyDeletionChargeApplies(src_url, src_obj_metadata, logger) MaybeSkipUnsupportedObject(src_url, src_obj_metadata) decryption_key = GetDecryptionCSEK(src_url, src_obj_metadata) src_obj_size = src_obj_metadata.size dst_obj_metadata.contentType = src_obj_metadata.contentType if global_copy_helper_opts.preserve_acl and dst_url.IsCloudUrl(): if src_url.scheme == 'gs' and not src_obj_metadata.acl: raise CommandException( 'No OWNER permission found for object %s. OWNER permission is ' 'required for preserving ACLs.' % src_url) dst_obj_metadata.acl = src_obj_metadata.acl # Special case for S3-to-S3 copy URLs using # global_copy_helper_opts.preserve_acl. # dst_url will be verified in _CopyObjToObjDaisyChainMode if it # is not s3 (and thus differs from src_url). if src_url.scheme == 's3': acl_text = S3MarkerAclFromObjectMetadata(src_obj_metadata) if acl_text: AddS3MarkerAclToObjectMetadata(dst_obj_metadata, acl_text) else: # src_url.IsFileUrl() if use_stet: source_stream_url = stet_util.encrypt_upload(src_url, dst_url, logger) else: source_stream_url = src_url try: src_obj_filestream = GetStreamFromFileUrl(source_stream_url) except Exception as e: # pylint: disable=broad-except message = 'Error opening file "%s": %s.' % (src_url, str(e)) if command_obj.continue_on_error: command_obj.op_failure_count += 1 logger.error(message) return else: raise CommandException(message) if src_url.IsStream() or src_url.IsFifo(): src_obj_size = None elif src_obj_metadata and src_obj_metadata.size and not use_stet: # Iterator retrieved the file's size, no need to stat it again. # Unless STET changed the file size. src_obj_size = src_obj_metadata.size else: src_obj_size = os.path.getsize(source_stream_url.object_name) if global_copy_helper_opts.use_manifest: # Set the source size in the manifest. manifest.Set(src_url.url_string, 'size', src_obj_size) if (dst_url.scheme == 's3' and src_url != 's3' and src_obj_size is not None and # Can't compare int to None in py3 src_obj_size > S3_MAX_UPLOAD_SIZE): raise CommandException( '"%s" exceeds the maximum gsutil-supported size for an S3 upload. S3 ' 'objects greater than %s in size require multipart uploads, which ' 'gsutil does not support.' % (src_url, MakeHumanReadable(S3_MAX_UPLOAD_SIZE))) # On Windows, stdin is opened as text mode instead of binary which causes # problems when piping a binary file, so this switches it to binary mode. if IS_WINDOWS and src_url.IsFileUrl() and src_url.IsStream(): msvcrt.setmode(GetStreamFromFileUrl(src_url).fileno(), os.O_BINARY) if global_copy_helper_opts.no_clobber: # There are two checks to prevent clobbering: # 1) The first check is to see if the URL # already exists at the destination and prevent the upload/download # from happening. This is done by the exists() call. # 2) The second check is only relevant if we are writing to gs. We can # enforce that the server only writes the object if it doesn't exist # by specifying the header below. This check only happens at the # server after the complete file has been uploaded. We specify this # header to prevent a race condition where a destination file may # be created after the first check and before the file is fully # uploaded. # In order to save on unnecessary uploads/downloads we perform both # checks. However, this may come at the cost of additional HTTP calls. if preconditions.gen_match: raise ArgumentException('Specifying x-goog-if-generation-match is ' 'not supported with cp -n') else: preconditions.gen_match = 0 if dst_url.IsFileUrl() and os.path.exists(dst_url.object_name): raise ItemExistsError() elif dst_url.IsCloudUrl(): try: dst_object = gsutil_api.GetObjectMetadata(dst_url.bucket_name, dst_url.object_name, provider=dst_url.scheme) except NotFoundException: dst_object = None if dst_object: raise ItemExistsError() if dst_url.IsCloudUrl(): # Cloud storage API gets object and bucket name from metadata. dst_obj_metadata.name = dst_url.object_name dst_obj_metadata.bucket = dst_url.bucket_name if src_url.IsCloudUrl(): # Preserve relevant metadata from the source object if it's not already # provided from the headers. src_obj_metadata.name = src_url.object_name src_obj_metadata.bucket = src_url.bucket_name else: _SetContentTypeFromFile(src_url, dst_obj_metadata) # Only set KMS key name if destination provider is 'gs'. encryption_keywrapper = GetEncryptionKeyWrapper(config) if (encryption_keywrapper and encryption_keywrapper.crypto_type == CryptoKeyType.CMEK and dst_url.scheme == 'gs'): dst_obj_metadata.kmsKeyName = encryption_keywrapper.crypto_key if src_obj_metadata: # Note that CopyObjectMetadata only copies specific fields. We intentionally # do not copy storageClass, as the bucket's default storage class should be # used (when copying to a gs:// bucket) unless explicitly overridden. CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False) if global_copy_helper_opts.dest_storage_class: dst_obj_metadata.storageClass = global_copy_helper_opts.dest_storage_class if config.get('GSUtil', 'check_hashes') == CHECK_HASH_NEVER: # GCS server will perform MD5 validation if the md5 hash is present. # Remove md5_hash if check_hashes=never. dst_obj_metadata.md5Hash = None _LogCopyOperation(logger, src_url, dst_url, dst_obj_metadata) if src_url.IsCloudUrl(): if dst_url.IsFileUrl(): PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_DOWNLOAD, size=src_obj_size, finished=False)) return _DownloadObjectToFile(src_url, src_obj_metadata, dst_url, gsutil_api, logger, command_obj, copy_exception_handler, allow_splitting=allow_splitting, decryption_key=decryption_key, is_rsync=is_rsync, preserve_posix=preserve_posix, use_stet=use_stet) elif copy_in_the_cloud: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_CLOUD_COPY, size=src_obj_size, finished=False)) return _CopyObjToObjInTheCloud(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, decryption_key=decryption_key) else: PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_DAISY_COPY, size=src_obj_size, finished=False)) return _CopyObjToObjDaisyChainMode(src_url, src_obj_metadata, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, decryption_key=decryption_key) else: # src_url.IsFileUrl() if dst_url.IsCloudUrl(): # The FileMessage for this upload object is inside _UploadFileToObject(). # This is such because the function may alter src_url, which would prevent # us from correctly tracking the new url. uploaded_metadata = _UploadFileToObject(src_url, src_obj_filestream, src_obj_size, dst_url, dst_obj_metadata, preconditions, gsutil_api, logger, command_obj, copy_exception_handler, gzip_exts=gzip_exts, allow_splitting=allow_splitting, gzip_encoded=gzip_encoded) if use_stet: # Delete temporary file. os.unlink(src_obj_filestream.name) return uploaded_metadata else: # dst_url.IsFileUrl() PutToQueueWithTimeout( gsutil_api.status_queue, FileMessage(src_url, dst_url, time.time(), message_type=FileMessage.FILE_LOCAL_COPY, size=src_obj_size, finished=False)) result = _CopyFileToFile(src_url, dst_url, status_queue=gsutil_api.status_queue, src_obj_metadata=src_obj_metadata) # Need to let _CopyFileToFile return before setting the POSIX attributes. if not src_url.IsStream() and not dst_url.IsStream(): ParseAndSetPOSIXAttributes(dst_url.object_name, src_obj_metadata, is_rsync=is_rsync, preserve_posix=preserve_posix) return result class Manifest(object): """Stores the manifest items for the CpCommand class.""" def __init__(self, path): # self.items contains a dictionary of rows self.items = {} self.manifest_filter = {} self.lock = parallelism_framework_util.CreateLock() self.manifest_path = os.path.expanduser(path) self._ParseManifest() self._CreateManifestFile() def _ParseManifest(self): """Load and parse a manifest file. This information will be used to skip any files that have a skip or OK status. """ try: if os.path.exists(self.manifest_path): # Note: we can't use io.open here or CSV reader will become upset # https://stackoverflow.com/a/18449496 with open(self.manifest_path, 'r') as f: first_row = True reader = csv.reader(f) for row in reader: if first_row: try: source_index = row.index('Source') result_index = row.index('Result') except ValueError: # No header and thus not a valid manifest file. raise CommandException('Missing headers in manifest file: %s' % self.manifest_path) first_row = False source = row[source_index] result = row[result_index] if result in ['OK', 'skip']: # We're always guaranteed to take the last result of a specific # source url. self.manifest_filter[source] = result except IOError: raise CommandException('Could not parse %s' % self.manifest_path) def WasSuccessful(self, src): """Returns whether the specified src url was marked as successful.""" return src in self.manifest_filter def _CreateManifestFile(self): """Opens the manifest file and assigns it to the file pointer.""" try: if ((not os.path.exists(self.manifest_path)) or (os.stat(self.manifest_path).st_size == 0)): # Add headers to the new file. if six.PY3: with open(self.manifest_path, 'w', newline='') as f: writer = csv.writer(f) writer.writerow([ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description', ]) else: with open(self.manifest_path, 'wb', 1) as f: writer = csv.writer(f) writer.writerow([ 'Source', 'Destination', 'Start', 'End', 'Md5', 'UploadId', 'Source Size', 'Bytes Transferred', 'Result', 'Description', ]) except IOError: raise CommandException('Could not create manifest file.') def Set(self, url, key, value): if value is None: # In case we don't have any information to set we bail out here. # This is so that we don't clobber existing information. # To zero information pass '' instead of None. return if url in self.items: self.items[url][key] = value else: self.items[url] = {key: value} def Initialize(self, source_url, destination_url): # Always use the source_url as the key for the item. This is unique. self.Set(source_url, 'source_uri', source_url) self.Set(source_url, 'destination_uri', destination_url) self.Set(source_url, 'start_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)) def SetResult(self, source_url, bytes_transferred, result, description=''): self.Set(source_url, 'bytes', bytes_transferred) self.Set(source_url, 'result', result) self.Set(source_url, 'description', description) self.Set(source_url, 'end_time', datetime.datetime.now(tz=datetime.timezone.utc).replace(tzinfo=None)) self._WriteRowToManifestFile(source_url) self._RemoveItemFromManifest(source_url) def _WriteRowToManifestFile(self, url): """Writes a manifest entry to the manifest file for the url argument.""" row_item = self.items[url] data = [ row_item['source_uri'], row_item['destination_uri'], '%sZ' % row_item['start_time'].isoformat(), '%sZ' % row_item['end_time'].isoformat(), row_item['md5'] if 'md5' in row_item else '', row_item['upload_id'] if 'upload_id' in row_item else '', str(row_item['size']) if 'size' in row_item else '', str(row_item['bytes']) if 'bytes' in row_item else '', row_item['result'], row_item['description'], ] data = [six.ensure_str(value) for value in data] # Aquire a lock to prevent multiple threads writing to the same file at # the same time. This would cause a garbled mess in the manifest file. with self.lock: if IS_WINDOWS and six.PY3: f = open(self.manifest_path, 'a', 1, newline='') else: f = open(self.manifest_path, 'a', 1) # 1 == line buffered writer = csv.writer(f) writer.writerow(data) f.close() def _RemoveItemFromManifest(self, url): # Remove the item from the dictionary since we're done with it and # we don't want the dictionary to grow too large in memory for no good # reason. del self.items[url] class ItemExistsError(Exception): """Exception class for objects that are skipped because they already exist.""" pass class SkipUnsupportedObjectError(Exception): """Exception for objects skipped because they are an unsupported type.""" def __init__(self): super(SkipUnsupportedObjectError, self).__init__() self.unsupported_type = 'Unknown' class SkipGlacierError(SkipUnsupportedObjectError): """Exception for objects skipped because they are an unsupported type.""" def __init__(self): super(SkipGlacierError, self).__init__() self.unsupported_type = 'GLACIER' def GetPathBeforeFinalDir(url, exp_src_url): """Returns the path section before the final directory component of the URL. This handles cases for file system directories, bucket, and bucket subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket', and for file://dir we'll return file:// Args: url: StorageUrl representing a filesystem directory, cloud bucket or bucket subdir. exp_src_url: StorageUrl representing the fully expanded object to-be-copied; used for resolving cloud wildcards. Returns: String name of above-described path, sans final path separator. """ sep = url.delim if url.IsFileUrl(): past_scheme = url.url_string[len('file://'):] if past_scheme.find(sep) == -1: return 'file://' else: return 'file://%s' % past_scheme.rstrip(sep).rpartition(sep)[0] if url.IsBucket(): return '%s://' % url.scheme # Else it names a bucket subdir. path_sans_final_dir = url.url_string.rstrip(sep).rpartition(sep)[0] return ResolveWildcardsInPathBeforeFinalDir(path_sans_final_dir, exp_src_url) def ResolveWildcardsInPathBeforeFinalDir(src_url_path_sans_final_dir, exp_src_url): """Returns the path section for a bucket subdir with wildcards resolved. This handles cases for bucket subdirectories where the initial source URL contains a wildcard. In this case, src_url must be wildcard-expanded before calculating the final directory. Example: A bucket containing: gs://bucket/dir1/subdir/foo gs://bucket/dir2/subdir/foo and source URL gs://bucket//subdir and src_url_path_sans_final dir gs://bucket/ should yield final path gs://bucket/dir1 or gs://bucket/dir2 according to the expanded source URL. Args: src_url_path_sans_final_dir: URL string with wildcards representing a bucket subdir as computed from GetPathBeforeFinalDir. exp_src_url: CloudUrl representing the fully expanded object to-be-copied. Returns: String name of above-described path, sans final path separator. """ if not ContainsWildcard(src_url_path_sans_final_dir): return src_url_path_sans_final_dir # Parse the expanded source URL, replacing wildcarded # portions of the path with what they actually expanded to. wildcarded_src_obj_path = StorageUrlFromString( src_url_path_sans_final_dir).object_name.split('/') expanded_src_obj_path = exp_src_url.object_name.split('/') for path_segment_index in range(len(wildcarded_src_obj_path)): if ContainsWildcard(wildcarded_src_obj_path[path_segment_index]): # The expanded path is guaranteed to be have at least as many path # segments as the wildcarded path. wildcarded_src_obj_path[path_segment_index] = ( expanded_src_obj_path[path_segment_index]) resolved_src_path = '/'.join(wildcarded_src_obj_path) final_path_url = exp_src_url.Clone() final_path_url.object_name = resolved_src_path return final_path_url.url_string def _GetPartitionInfo(file_size, max_components, default_component_size): """Gets info about a file partition for parallel file/object transfers. Args: file_size: The number of bytes in the file to be partitioned. max_components: The maximum number of components that can be composed. default_component_size: The size of a component, assuming that max_components is infinite. Returns: The number of components in the partitioned file, and the size of each component (except the last, which will have a different size iff file_size != 0 (mod num_components)). """ # num_components = ceil(file_size / default_component_size) num_components = DivideAndCeil(file_size, default_component_size) # num_components must be in the range [2, max_components] num_components = max(min(num_components, max_components), 2) # component_size = ceil(file_size / num_components) component_size = DivideAndCeil(file_size, num_components) return (num_components, component_size) def _DeleteTempComponentObjectFn(cls, url_to_delete, thread_state=None): """Wrapper func to be used with command.Apply to delete temporary objects.""" gsutil_api = GetCloudApiInstance(cls, thread_state) try: gsutil_api.DeleteObject(url_to_delete.bucket_name, url_to_delete.object_name, generation=url_to_delete.generation, provider=url_to_delete.scheme) except NotFoundException: # The temporary object could already be gone if a retry was # issued at a lower layer but the original request succeeded. # Barring other errors, the top-level command should still report success, # so don't raise here. pass def FilterExistingComponents(dst_args, existing_components, bucket_url, gsutil_api): """Determines course of action for component objects. Given the list of all target objects based on partitioning the file and the list of objects that have already been uploaded successfully, this function determines which objects should be uploaded, which existing components are still valid, and which existing components should be deleted. Args: dst_args: The map of file_name -> PerformParallelUploadFileToObjectArgs calculated by partitioning the file. existing_components: A list of ObjectFromTracker objects that have been uploaded in the past. bucket_url: CloudUrl of the bucket in which the components exist. gsutil_api: gsutil Cloud API instance to use for retrieving object metadata. Returns: components_to_upload: List of components that need to be uploaded. uploaded_components: List of components that have already been uploaded and are still valid. Each element of the list contains the dst_url for the uploaded component and its size. existing_objects_to_delete: List of components that have already been uploaded, but are no longer valid and are in a versioned bucket, and therefore should be deleted. """ components_to_upload = [] existing_component_names = [ component.object_name for component in existing_components ] for component_name in dst_args: if component_name not in existing_component_names: components_to_upload.append(dst_args[component_name]) objects_already_chosen = [] # Don't reuse any temporary components whose MD5 doesn't match the current # MD5 of the corresponding part of the file. If the bucket is versioned, # also make sure that we delete the existing temporary version. existing_objects_to_delete = [] uploaded_components = [] for tracker_object in existing_components: if (tracker_object.object_name not in dst_args.keys() or tracker_object.object_name in objects_already_chosen): # This could happen if the component size has changed. This also serves # to handle object names that get duplicated in the tracker file due # to people doing things they shouldn't (e.g., overwriting an existing # temporary component in a versioned bucket). url = bucket_url.Clone() url.object_name = tracker_object.object_name url.generation = tracker_object.generation existing_objects_to_delete.append(url) continue dst_arg = dst_args[tracker_object.object_name] file_part = FilePart(dst_arg.filename, dst_arg.file_start, dst_arg.file_length) # TODO: calculate MD5's in parallel when possible. content_md5 = CalculateB64EncodedMd5FromContents(file_part) try: # Get the MD5 of the currently-existing component. dst_url = dst_arg.dst_url dst_metadata = gsutil_api.GetObjectMetadata( dst_url.bucket_name, dst_url.object_name, generation=dst_url.generation, provider=dst_url.scheme, fields=['customerEncryption', 'etag', 'md5Hash']) cloud_md5 = dst_metadata.md5Hash except Exception: # pylint: disable=broad-except # We don't actually care what went wrong - we couldn't retrieve the # object to check the MD5, so just upload it again. cloud_md5 = None if cloud_md5 != content_md5: components_to_upload.append(dst_arg) objects_already_chosen.append(tracker_object.object_name) if tracker_object.generation: # If the old object doesn't have a generation (i.e., it isn't in a # versioned bucket), then we will just overwrite it anyway. invalid_component_with_generation = dst_arg.dst_url.Clone() invalid_component_with_generation.generation = tracker_object.generation existing_objects_to_delete.append(invalid_component_with_generation) else: url = dst_arg.dst_url.Clone() url.generation = tracker_object.generation uploaded_components.append((url, dst_arg.file_length)) objects_already_chosen.append(tracker_object.object_name) if uploaded_components: logging.info('Found %d existing temporary components to reuse.', len(uploaded_components)) return (components_to_upload, uploaded_components, existing_objects_to_delete)	181.74 KB	2025-06-25 08:19
📄 encryption_helper.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for dealing with encryption keys used with cloud APIs.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import binascii from hashlib import sha256 import re import sys import six from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper MAX_DECRYPTION_KEYS = 100 VALID_CMEK_RE = LazyWrapper( lambda: re.compile('projects/([^/]+)/' 'locations/([a-zA-Z0-9_-]{1,63})/' 'keyRings/([a-zA-Z0-9_-]{1,63})/' 'cryptoKeys/([a-zA-Z0-9_-]{1,63})$')) class CryptoKeyType(object): """Enum of valid types of encryption keys used with cloud API requests.""" CSEK = 'CSEK' CMEK = 'CMEK' class CryptoKeyWrapper(object): """Class describing a crypto key used with cloud API requests. This class should be instantiated via the `CryptoKeyWrapperFromKey` method. """ def __init__(self, crypto_key): """Initialize the CryptoKeyWrapper. Args: crypto_key: Base64-encoded string of a CSEK, or the name of a Cloud KMS CMEK. Raises: CommandException: The specified crypto key was neither a CMEK key name nor a valid base64-encoded string of a CSEK. """ self.crypto_key = crypto_key # Base64-encoded CSEKs always have a length of 44 characters, whereas # fully-qualified CMEK names are guaranteed to be longer than 45 characters. if len(crypto_key) == 44: self.crypto_type = CryptoKeyType.CSEK self.crypto_alg = 'AES256' # Only currently supported algorithm for CSEK. try: self.crypto_key_sha256 = Base64Sha256FromBase64EncryptionKey(crypto_key) except: raise CommandException( 'Configured encryption_key or decryption_key looked like a CSEK, ' 'but it was not a valid 44-character base64 string. Please ' 'double-check your configuration and ensure the key is correct.') else: # CMEK try: ValidateCMEK(crypto_key) except CommandException as e: raise CommandException( 'Configured encryption_key or decryption_key looked like a CMEK, ' 'but the key failed validation:\n%s' % e.reason) self.crypto_type = CryptoKeyType.CMEK self.crypto_alg = None self.crypto_key_sha256 = None def CryptoKeyWrapperFromKey(crypto_key): """Returns a CryptoKeyWrapper for crypto_key, or None for no key.""" return CryptoKeyWrapper(crypto_key) if crypto_key else None def FindMatchingCSEKInBotoConfig(key_sha256, boto_config): """Searches boto_config for a CSEK with the given base64-encoded SHA256 hash. Args: key_sha256: (str) Base64-encoded SHA256 hash of the AES256 encryption key. boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string if a match is found, None otherwise. """ if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') keywrapper = CryptoKeyWrapperFromKey( boto_config.get('GSUtil', 'encryption_key', None)) if (keywrapper is not None and keywrapper.crypto_type == CryptoKeyType.CSEK and keywrapper.crypto_key_sha256 == key_sha256): return keywrapper.crypto_key for i in range(MAX_DECRYPTION_KEYS): key_number = i + 1 keywrapper = CryptoKeyWrapperFromKey( boto_config.get('GSUtil', 'decryption_key%s' % str(key_number), None)) if keywrapper is None: # Reading 100 config values can take ~1ms in testing. To avoid adding # this tax, stop reading keys as soon as we encounter a non-existent # entry (in lexicographic order). break elif (keywrapper.crypto_type == CryptoKeyType.CSEK and keywrapper.crypto_key_sha256 == key_sha256): return keywrapper.crypto_key def GetEncryptionKeyWrapper(boto_config): """Returns a CryptoKeyWrapper for the configured encryption key. Reads in the value of the "encryption_key" attribute in boto_config, and if present, verifies it is a valid base64-encoded string and returns a CryptoKeyWrapper for it. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: CryptoKeyWrapper for the specified encryption key, or None if no encryption key was specified in boto_config. """ encryption_key = boto_config.get('GSUtil', 'encryption_key', None) return CryptoKeyWrapper(encryption_key) if encryption_key else None def Base64Sha256FromBase64EncryptionKey(csek_encryption_key): if six.PY3: if not isinstance(csek_encryption_key, bytes): csek_encryption_key = csek_encryption_key.encode('ascii') decoded_bytes = base64.b64decode(csek_encryption_key) key_sha256 = _CalculateSha256FromString(decoded_bytes) sha256_bytes = binascii.unhexlify(key_sha256) sha256_base64 = base64.b64encode(sha256_bytes) return sha256_base64.replace(b'\n', b'') def ValidateCMEK(key): if not key: raise CommandException('KMS key is empty.') if key.startswith('/'): raise CommandException( 'KMS key should not start with leading slash (/): "%s"' % key) if not VALID_CMEK_RE().match(key): raise CommandException( 'Invalid KMS key name: "%s".\nKMS keys should follow the format ' '"projects/<project-id>/locations/<location>/keyRings/<keyring>/' 'cryptoKeys/<key-name>"' % key) def _CalculateSha256FromString(input_string): sha256_hash = sha256() sha256_hash.update(input_string) return sha256_hash.hexdigest() def _GetAndVerifyBase64EncryptionKey(boto_config): """Reads the encryption key from boto_config and ensures it is base64-encoded. Args: boto_config: (boto.pyami.config.Config) The boto config in which to check for a matching encryption key. Returns: (str) Base64-encoded encryption key string, or None if no encryption key exists in configuration. """ encryption_key = boto_config.get('GSUtil', 'encryption_key', None) if encryption_key: # Ensure the key has a valid encoding. try: base64.b64decode(encryption_key) except: raise CommandException( 'Configured encryption_key is not a valid base64 string. Please ' 'double-check your configuration and ensure the key is valid and in ' 'base64 format.') return encryption_key	7 KB	2025-06-25 08:19
📄 execution_util.py # -- coding: utf-8 -- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for executing binaries.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import subprocess from gslib import exception def ExecuteExternalCommand(command_and_flags): """Runs external terminal command. Args: command_and_flags (List[str]): Ordered command and flag strings. Returns: (stdout (str\|None), stderr (str\|None)) from running command. Raises: OSError for any issues running the command. """ command_process = subprocess.Popen(command_and_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE) command_stdout, command_stderr = command_process.communicate() # Python 3 outputs bytes from communicate() by default. if command_stdout is not None and not isinstance(command_stdout, str): command_stdout = command_stdout.decode() if command_stderr is not None and not isinstance(command_stderr, str): command_stderr = command_stderr.decode() if command_process.returncode != 0: raise exception.ExternalBinaryError(command_stderr) return command_stdout, command_stderr	1.8 KB	2025-06-25 08:19
📄 hashing_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for hashing functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import binascii import hashlib import os import six from boto import config import crcmod from gslib.exception import CommandException from gslib.utils.boto_util import UsingCrcmodExtension from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE from gslib.utils.constants import MIN_SIZE_COMPUTE_LOGGING from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.constants import UTF8 SLOW_CRCMOD_WARNING = """ WARNING: You have requested checksumming but your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. For help installing the extension, please see "gsutil help crcmod". """ SLOW_CRCMOD_RSYNC_WARNING = """ WARNING: gsutil rsync uses hashes when modification time is not available at both the source and destination. Your crcmod installation isn't using the module's C extension, so checksumming will run very slowly. If this is your first rsync since updating gsutil, this rsync can take significantly longer than usual. For help installing the extension, please see "gsutil help crcmod". """ _SLOW_CRCMOD_DOWNLOAD_WARNING = """ WARNING: Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To disable slow integrity checking, see the "check_hashes" option in your boto config file. """ _SLOW_CRC_EXCEPTION_TEXT = """ Downloading this composite object requires integrity checking with CRC32c, but your crcmod installation isn't using the module's C extension, so the hash computation will likely throttle download performance. For help installing the extension, please see "gsutil help crcmod". To download regardless of crcmod performance or to skip slow integrity checks, see the "check_hashes" option in your boto config file. NOTE: It is strongly recommended that you not disable integrity checks. Doing so could allow data corruption to go undetected during uploading/downloading.""" _NO_HASH_CHECK_WARNING = """ WARNING: This download will not be validated since your crcmod installation doesn't use the module's C extension, so the hash computation would likely throttle download performance. For help in installing the extension, please see "gsutil help crcmod". To force integrity checking, see the "check_hashes" option in your boto config file. """ # Configuration values for hashing. CHECK_HASH_IF_FAST_ELSE_FAIL = 'if_fast_else_fail' CHECK_HASH_IF_FAST_ELSE_SKIP = 'if_fast_else_skip' CHECK_HASH_ALWAYS = 'always' CHECK_HASH_NEVER = 'never' # Table storing polynomial values of x^(2^k) mod CASTAGNOLI_POLY for all k < 31, # where x^(2^k) and CASTAGNOLI_POLY are both considered polynomials. This is # sufficient since x^(2^31) mod CASTAGNOLI_POLY = x. X_POW_2K_TABLE = [ 2, 4, 16, 256, 65536, 517762881, 984302966, 408362264, 1503875210, 2862076957, 3884826397, 1324787473, 621200174, 1758783527, 1416537776, 1180494764, 648569364, 2521473789, 994858823, 1728245375, 3498467999, 4059169852, 3345064394, 2828422810, 2429203150, 3336788029, 860151998, 2102628683, 1033187991, 4243778976, 1123580069 ] # Castagnoli polynomial and its degree. CASTAGNOLI_POLY = 4812730177 DEGREE = 32 def ConcatCrc32c(crc_a, crc_b, num_bytes_in_b): """Computes CRC32C for concat(A, B) given crc(A), crc(B) and len(B). An explanation of the algorithm can be found at crcutil.googlecode.com/files/crc-doc.1.0.pdf. Args: crc_a: A 32-bit integer representing crc(A) with least-significant coefficient first. crc_b: Same as crc_a. num_bytes_in_b: Length of B in bytes. Returns: CRC32C for concat(A, B) """ if not num_bytes_in_b: return crc_a return _ExtendByZeros(crc_a, 8 * num_bytes_in_b) ^ crc_b def _CrcMultiply(p, q): """Multiplies two polynomials together modulo CASTAGNOLI_POLY. Args: p: The first polynomial. q: The second polynomial. Returns: Result of the multiplication. """ result = 0 top_bit = 1 << DEGREE for _ in range(DEGREE): if p & 1: result ^= q q <<= 1 if q & top_bit: q ^= CASTAGNOLI_POLY p >>= 1 return result def _ExtendByZeros(crc, num_bits): """Given crc representing polynomial P(x), compute P(x)x^num_bits. Args: crc: crc respresenting polynomial P(x). num_bits: number of bits in crc. Returns: P(x)x^num_bits """ def _ReverseBits32(crc): return int('{0:032b}'.format(crc, width=32)[::-1], 2) crc = _ReverseBits32(crc) i = 0 while num_bits != 0: if num_bits & 1: crc = _CrcMultiply(crc, X_POW_2K_TABLE[i % len(X_POW_2K_TABLE)]) i += 1 num_bits >>= 1 crc = _ReverseBits32(crc) return crc def _CalculateHashFromContents(fp, hash_alg): """Calculates a base64 digest of the contents of a seekable stream. This function resets the file pointer to position 0. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in hex string format. """ hash_dict = {'placeholder': hash_alg} fp.seek(0) CalculateHashesFromContents(fp, hash_dict) fp.seek(0) return hash_dict['placeholder'].hexdigest() def CalculateHashesFromContents(fp, hash_dict, callback_processor=None): """Calculates hashes of the contents of a file. Args: fp: An already-open file object (stream will be consumed). hash_dict: Dict of (string alg_name: initialized hashing class) Hashing class will be populated with digests upon return. callback_processor: Optional callback processing class that implements Progress(integer amount of bytes processed). """ while True: data = fp.read(DEFAULT_FILE_BUFFER_SIZE) if not data: break if six.PY3: if isinstance(data, str): data = data.encode(UTF8) for hash_alg in six.itervalues(hash_dict): hash_alg.update(data) if callback_processor: callback_processor.Progress(len(data)) def CalculateB64EncodedCrc32cFromContents(fp): """Calculates a base64 CRC32c checksum of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: CRC32c checksum of the file in base64 format. """ return _CalculateB64EncodedHashFromContents(fp, crcmod.predefined.Crc('crc-32c')) def CalculateB64EncodedMd5FromContents(fp): """Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in base64 format. """ return _CalculateB64EncodedHashFromContents(fp, GetMd5()) def CalculateMd5FromContents(fp): """Calculates a base64 MD5 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. Returns: MD5 digest of the file in hex format. """ return _CalculateHashFromContents(fp, GetMd5()) def Base64EncodeHash(digest_value): """Returns the base64-encoded version of the input hex digest value.""" encoded_bytes = base64.b64encode(binascii.unhexlify(digest_value)) return encoded_bytes.rstrip(b'\n').decode(UTF8) def Base64ToHexHash(base64_hash): """Returns the hex digest value of the input base64-encoded hash. Args: base64_hash: Base64-encoded hash, which may contain newlines and single or double quotes. Returns: Hex digest of the input argument. """ decoded_bytes = base64.b64decode(base64_hash.strip('\n"\'').encode(UTF8)) return binascii.hexlify(decoded_bytes) def _CalculateB64EncodedHashFromContents(fp, hash_alg): """Calculates a base64 digest of the contents of a seekable stream. This function sets the stream position 0 before and after calculation. Args: fp: An already-open file object. hash_alg: Instance of hashing class initialized to start state. Returns: Hash of the stream in base64 format. """ return Base64EncodeHash(_CalculateHashFromContents(fp, hash_alg)) def GetUploadHashAlgs(): """Returns a dict of hash algorithms for validating an uploaded object. This is for use only with single object uploads, not compose operations such as those used by parallel composite uploads (though it can be used to validate the individual components). Returns: dict of (algorithm_name: hash_algorithm) """ check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == 'never': return {} return {'md5': GetMd5} def GetDownloadHashAlgs(logger, consider_md5=False, consider_crc32c=False): """Returns a dict of hash algorithms for validating an object. Args: logger: logging.Logger for outputting log messages. consider_md5: If True, consider using a md5 hash. consider_crc32c: If True, consider using a crc32c hash. Returns: Dict of (string, hash algorithm). Raises: CommandException if hash algorithms satisfying the boto config file cannot be returned. """ check_hashes_config = config.get('GSUtil', 'check_hashes', CHECK_HASH_IF_FAST_ELSE_FAIL) if check_hashes_config == CHECK_HASH_NEVER: return {} hash_algs = {} if consider_md5: hash_algs['md5'] = GetMd5 elif consider_crc32c: # If the cloud provider supplies a CRC, we'll compute a checksum to # validate if we're using a native crcmod installation and MD5 isn't # offered as an alternative. if UsingCrcmodExtension(): hash_algs['crc32c'] = lambda: crcmod.predefined.Crc('crc-32c') elif not hash_algs: if check_hashes_config == CHECK_HASH_IF_FAST_ELSE_FAIL: raise CommandException(_SLOW_CRC_EXCEPTION_TEXT) elif check_hashes_config == CHECK_HASH_IF_FAST_ELSE_SKIP: logger.warn(_NO_HASH_CHECK_WARNING) elif check_hashes_config == CHECK_HASH_ALWAYS: logger.warn(_SLOW_CRCMOD_DOWNLOAD_WARNING) hash_algs['crc32c'] = lambda: crcmod.predefined.Crc('crc-32c') else: raise CommandException( 'Your boto config \'check_hashes\' option is misconfigured.') return hash_algs def GetMd5(byte_string=b''): """Returns md5 object, avoiding incorrect FIPS error on Red Hat systems. Examples: GetMd5(b'abc') GetMd5(bytes('abc', encoding='utf-8')) Args: byte_string (bytes): String in bytes form to hash. Don't include for empty hash object, since md5(b'').digest() == md5().digest(). Returns: md5 hash object. """ try: return hashlib.md5(byte_string) except ValueError: # On Red Hat-based platforms, may catch a FIPS error. # "usedforsecurity" flag only available on Red Hat systems or Python 3.9+. # pylint:disable=unexpected-keyword-arg return hashlib.md5(byte_string, usedforsecurity=False) # pylint:enable=unexpected-keyword-arg class HashingFileUploadWrapper(object): """Wraps an input stream in a hash digester and exposes a stream interface. This class provides integrity checking during file uploads via the following properties: Calls to read will appropriately update digesters with all bytes read. Calls to seek (assuming it is supported by the wrapped stream) using os.SEEK_SET will catch up / reset the digesters to the specified position. If seek is called with a different os.SEEK mode, the caller must return to the original position using os.SEEK_SET before further reads. Calls to seek are fast if the desired position is equal to the position at the beginning of the last read call (we only need to re-hash bytes from that point on). """ def __init__(self, stream, digesters, hash_algs, src_url, logger): """Initializes the wrapper. Args: stream: Input stream. digesters: dict of {string: hash digester} containing digesters, where string is the name of the hash algorithm. hash_algs: dict of {string: hash algorithm} for resetting and recalculating digesters. String is the name of the hash algorithm. src_url: Source FileUrl that is being copied. logger: For outputting log messages. """ if not digesters: raise CommandException('HashingFileUploadWrapper used with no digesters.') elif not hash_algs: raise CommandException('HashingFileUploadWrapper used with no hash_algs.') self._orig_fp = stream self._digesters = digesters self._src_url = src_url self._logger = logger self._seek_away = None self._digesters_previous = {} for alg in self._digesters: self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters_previous_mark = 0 self._digesters_current_mark = 0 self._hash_algs = hash_algs @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def read(self, size=-1): # pylint: disable=invalid-name """"Reads from the wrapped file pointer and calculates hash digests. Args: size: The amount of bytes to read. If ommited or negative, the entire contents of the file will be read, hashed, and returned. Returns: Bytes from the wrapped stream. Raises: CommandException if the position of the wrapped stream is unknown. """ if self._seek_away is not None: raise CommandException('Read called on hashing file pointer in an ' 'unknown position; cannot correctly compute ' 'digest.') data = self._orig_fp.read(size) if isinstance(data, six.text_type): data = data.encode(UTF8) self._digesters_previous_mark = self._digesters_current_mark for alg in self._digesters: self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters[alg].update(data) self._digesters_current_mark += len(data) return data def tell(self): # pylint: disable=invalid-name """Returns the current stream position.""" return self._orig_fp.tell() def seekable(self): # pylint: disable=invalid-name """Returns true if the stream is seekable.""" return self._orig_fp.seekable() def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Seeks in the wrapped file pointer and catches up hash digests. Args: offset: The offset to seek to. whence: os.SEEK_CUR, or SEEK_END, SEEK_SET. Returns: Return value from the wrapped stream's seek call. """ if whence != os.SEEK_SET: # We do not catch up hashes for non-absolute seeks, and rely on the # caller to seek to an absolute position before reading. self._seek_away = self._orig_fp.tell() else: # Hashes will be correct and it's safe to call read(). self._seek_away = None if offset < self._digesters_previous_mark: # This is earlier than our earliest saved digest, so we need to # reset the digesters and scan from the beginning. for alg in self._digesters: self._digesters[alg] = self._hash_algs[alg]() self._digesters_current_mark = 0 self._orig_fp.seek(0) self._CatchUp(offset) elif offset == self._digesters_previous_mark: # Just load the saved digests. self._digesters_current_mark = self._digesters_previous_mark for alg in self._digesters: self._digesters[alg] = self._digesters_previous[alg] elif offset < self._digesters_current_mark: # Reset the position to our previous digest and scan forward. self._digesters_current_mark = self._digesters_previous_mark for alg in self._digesters: self._digesters[alg] = self._digesters_previous[alg] self._orig_fp.seek(self._digesters_previous_mark) self._CatchUp(offset - self._digesters_previous_mark) else: # Scan forward from our current digest and position. self._orig_fp.seek(self._digesters_current_mark) self._CatchUp(offset - self._digesters_current_mark) return self._orig_fp.seek(offset, whence) def _CatchUp(self, bytes_to_read): """Catches up hashes, but does not return data and uses little memory. Before calling this function, digesters_current_mark should be updated to the current location of the original stream and the self._digesters should be current to that point (but no further). Args: bytes_to_read: Number of bytes to catch up from the original stream. """ if self._orig_fp.tell() != self._digesters_current_mark: raise CommandException( 'Invalid mark when catching up hashes. Stream position %s, hash ' 'position %s' % (self._orig_fp.tell(), self._digesters_current_mark)) for alg in self._digesters: if bytes_to_read >= MIN_SIZE_COMPUTE_LOGGING: self._logger.debug('Catching up %s for %s...', alg, self._src_url.url_string) self._digesters_previous[alg] = self._digesters[alg].copy() self._digesters_previous_mark = self._digesters_current_mark bytes_remaining = bytes_to_read bytes_this_round = min(bytes_remaining, TRANSFER_BUFFER_SIZE) while bytes_this_round: data = self._orig_fp.read(bytes_this_round) if isinstance(data, six.text_type): data = data.encode(UTF8) bytes_remaining -= bytes_this_round for alg in self._digesters: self._digesters[alg].update(data) bytes_this_round = min(bytes_remaining, TRANSFER_BUFFER_SIZE) self._digesters_current_mark += bytes_to_read	18.32 KB	2025-06-25 08:19
📄 iam_helper.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper module for the IAM command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import defaultdict from collections import namedtuple import six from apitools.base.protorpclite import protojson from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages TYPES = set([ 'user', 'deleted:user', 'serviceAccount', 'deleted:serviceAccount', 'group', 'deleted:group', 'domain', 'principal', 'principalSet', 'principalHierarchy' ]) DISCOURAGED_TYPES = set([ 'projectOwner', 'projectEditor', 'projectViewer', ]) DISCOURAGED_TYPES_MSG = ( 'Assigning roles (e.g. objectCreator, legacyBucketOwner) for project ' 'convenience groups is not supported by gsutil, as it goes against the ' 'principle of least privilege. Consider creating and using more granular ' 'groups with which to assign permissions. See ' 'https://cloud.google.com/iam/docs/using-iam-securely for more ' 'information. Assigning a role to a project group can be achieved by ' 'setting the IAM policy directly (see gsutil help iam for specifics).') PUBLIC_MEMBERS = set([ 'allUsers', 'allAuthenticatedUsers', ]) # These are convenience classes to handle returned results from # BindingStringToTuple. is_grant is a boolean specifying if the # bindings are to be granted or removed from a bucket / object. # bindings varies by type. # For BindingsTuple, it is a list of BindingsValueListEntry # For BindingsDictTuple it is a list of analogous dicts, e.g.: # { # 'members': ['member'], # 'role': 'role', # } BindingsTuple = namedtuple('BindingsTuple', ['is_grant', 'bindings']) BindingsDictTuple = namedtuple('BindingsDictTuple', ['is_grant', 'bindings']) # This is a special role value assigned to a specific member when all roles # assigned to the member should be dropped in the policy. A member:DROP_ALL # binding will be passed from BindingStringToTuple into PatchBindings. # This role will only ever appear on client-side (i.e. user-generated). It # will never be returned as a real role from an IAM get request. All roles # returned by PatchBindings are guaranteed to be "real" roles, i.e. not a # DROP_ALL role. DROP_ALL = '' def SerializeBindingsTuple(bindings_tuple): """Serializes the BindingsValueListEntry instances in a BindingsTuple. This is necessary when passing instances of BindingsTuple through Command.Apply, as apitools_messages classes are not by default pickleable. Args: bindings_tuple: A BindingsTuple instance to be serialized. Returns: A serialized BindingsTuple object. """ return (bindings_tuple.is_grant, [protojson.encode_message(t) for t in bindings_tuple.bindings]) def DeserializeBindingsTuple(serialized_bindings_tuple): (is_grant, bindings) = serialized_bindings_tuple return BindingsTuple(is_grant=is_grant, bindings=[ protojson.decode_message( apitools_messages.Policy.BindingsValueListEntry, t) for t in bindings ]) def BindingsMessageToUpdateDict(bindings): """Reformats policy bindings metadata. Args: bindings: A list of BindingsValueListEntry instances. Returns: A {role: set(members)} dictionary. """ tmp_bindings = defaultdict(set) for binding in bindings: tmp_bindings[binding.role].update(binding.members) return tmp_bindings def BindingsDictToUpdateDict(bindings): """Reformats policy bindings metadata. Args: bindings: List of dictionaries representing BindingsValueListEntry instances. e.g.: { "role": "some_role", "members": ["allAuthenticatedUsers", ...] } Returns: A {role: set(members)} dictionary. """ tmp_bindings = defaultdict(set) for binding in bindings: tmp_bindings[binding['role']].update(binding['members']) return tmp_bindings def IsEqualBindings(a, b): (granted, removed) = DiffBindings(a, b) return not granted.bindings and not removed.bindings def DiffBindings(old, new): """Computes the difference between two BindingsValueListEntry lists. Args: old: The original list of BindingValuesListEntry instances new: The updated list of BindingValuesListEntry instances Returns: A pair of BindingsTuple instances, one for roles granted between old and new, and one for roles removed between old and new. """ tmp_old = BindingsMessageToUpdateDict(old) tmp_new = BindingsMessageToUpdateDict(new) granted = BindingsMessageToUpdateDict([]) removed = BindingsMessageToUpdateDict([]) for (role, members) in six.iteritems(tmp_old): removed[role].update(members.difference(tmp_new[role])) for (role, members) in six.iteritems(tmp_new): granted[role].update(members.difference(tmp_old[role])) granted = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(granted) if m ] removed = [ apitools_messages.Policy.BindingsValueListEntry(role=r, members=list(m)) for (r, m) in six.iteritems(removed) if m ] return (BindingsTuple(True, granted), BindingsTuple(False, removed)) def PatchBindings(base, diff, is_grant): """Patches a diff list of BindingsValueListEntry to the base. Will remove duplicate members for any given role on a grant operation. Args: base (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing a resource's current IAM policy. diff (dict): A dictionary returned by BindingsMessageToUpdateDict or BindingsDictToUpdateDict representing the IAM policy bindings to add/remove from `base`. is_grant (bool): True if `diff` should be added to `base`, False if it should be removed from `base`. Returns: A {role: set(members)} dictionary created by applying `diff` to `base`. """ # Patch the diff into base if is_grant: for (role, members) in six.iteritems(diff): if not role: raise CommandException('Role must be specified for a grant request.') base[role].update(members) else: for role in base: base[role].difference_update(diff[role]) # Drop all members with the DROP_ALL role specifed from input. base[role].difference_update(diff[DROP_ALL]) return {role: members for role, members in six.iteritems(base) if members} def BindingStringToTuple(is_grant, input_str): """Parses an iam ch bind string to a list of binding tuples. Args: is_grant: If true, binding is to be appended to IAM policy; else, delete this binding from the policy. input_str: A string representing a member-role binding. e.g. user:foo@bar.com:objectAdmin user:foo@bar.com:objectAdmin,objectViewer user:foo@bar.com allUsers deleted:user:foo@bar.com?uid=123:objectAdmin,objectViewer deleted:serviceAccount:foo@bar.com?uid=123 Raises: CommandException in the case of invalid input. Returns: A BindingsDictTuple instance. """ if not input_str.count(':'): input_str += ':' # Allows user specified PUBLIC_MEMBERS, DISCOURAGED_TYPES, and TYPES to be # case insensitive. tokens = input_str.split(":") public_members = {s.lower(): s for s in PUBLIC_MEMBERS} types = {s.lower(): s for s in TYPES} discouraged_types = {s.lower(): s for s in DISCOURAGED_TYPES} possible_public_member_or_type = tokens[0].lower() possible_type = '%s:%s' % (tokens[0].lower(), tokens[1].lower()) if possible_public_member_or_type in public_members: tokens[0] = public_members[possible_public_member_or_type] elif possible_public_member_or_type in types: tokens[0] = types[possible_public_member_or_type] elif possible_public_member_or_type in discouraged_types: tokens[0] = discouraged_types[possible_public_member_or_type] elif possible_type in types: (tokens[0], tokens[1]) = types[possible_type].split(':') input_str = ":".join(tokens) # We can remove project convenience members, but not add them. removing_discouraged_type = not is_grant and tokens[0] in DISCOURAGED_TYPES if input_str.count(':') == 1: if '%s:%s' % (tokens[0], tokens[1]) in TYPES: raise CommandException('Incorrect public member type for binding %s' % input_str) elif tokens[0] in PUBLIC_MEMBERS: (member, roles) = tokens elif tokens[0] in TYPES or removing_discouraged_type: member = input_str roles = DROP_ALL else: raise CommandException('Incorrect public member type for binding %s' % input_str) elif input_str.count(':') == 2: if '%s:%s' % (tokens[0], tokens[1]) in TYPES: # case "deleted:user:foo@bar.com?uid=1234" member = input_str roles = DROP_ALL elif removing_discouraged_type: (member_type, project_id, roles) = tokens member = '%s:%s' % (member_type, project_id) else: (member_type, member_id, roles) = tokens _check_member_type(member_type, input_str) member = '%s:%s' % (member_type, member_id) elif input_str.count(':') == 3: # case "deleted:user:foo@bar.com?uid=1234:objectAdmin,objectViewer" (member_type_p1, member_type_p2, member_id, roles) = input_str.split(':') member_type = '%s:%s' % (member_type_p1, member_type_p2) _check_member_type(member_type, input_str) member = '%s:%s' % (member_type, member_id) else: raise CommandException('Invalid ch format %s' % input_str) if is_grant and not roles: raise CommandException('Must specify a role to grant.') roles = [ResolveRole(r) for r in roles.split(',')] bindings = [{'role': r, 'members': [member]} for r in set(roles)] return BindingsDictTuple(is_grant=is_grant, bindings=bindings) def _check_member_type(member_type, input_str): if member_type in DISCOURAGED_TYPES: raise CommandException(DISCOURAGED_TYPES_MSG) elif member_type not in TYPES: raise CommandException('Incorrect member type for binding %s' % input_str) def ResolveRole(role): if not role: return DROP_ALL if 'roles/' in role: return role return 'roles/storage.%s' % role	10.82 KB	2025-06-25 08:19
📄 ls_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions and class for listing commands such as ls and du.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import fnmatch import sys import six from gslib.cloud_api import EncryptionException from gslib.exception import CommandException from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.storage_url import GenerationFromUrlAndString from gslib.utils.constants import S3_ACL_MARKER_GUID from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import S3_MARKER_GUIDS from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AclTranslation from gslib.utils import text_util from gslib.wildcard_iterator import StorageUrlFromString ENCRYPTED_FIELDS = [ 'md5Hash', 'crc32c', ] UNENCRYPTED_FULL_LISTING_FIELDS = [ 'acl', 'cacheControl', 'componentCount', 'contentDisposition', 'contentEncoding', 'contentLanguage', 'contentType', 'customTime', 'kmsKeyName', 'customerEncryption', 'etag', 'eventBasedHold', 'generation', 'metadata', 'metageneration', 'retentionExpirationTime', 'size', 'storageClass', 'temporaryHold', 'timeCreated', 'timeDeleted', 'timeStorageClassUpdated', 'updated', ] def MakeMetadataLine(label, value, indent=1): """Returns a string with a vertically aligned label and value. Labels of the same indentation level will start at the same column. Values will all start at the same column (unless the combined left-indent and label length is excessively long). If a value spans multiple lines, indentation will only be applied to the first line. Example output from several calls: Label1: Value (default indent of 1 was used) Sublabel1: Value (used indent of 2 here) Label2: Value Args: label: The label to print in the first column. value: The value to print in the second column. indent: (4 * indent) spaces will be placed before the label. Returns: A string with a vertically aligned label and value. """ return '{}{}'.format(((' ' * indent * 4) + label + ':').ljust(28), value) def PrintBucketHeader(bucket_listing_ref): # pylint: disable=unused-argument """Default function for printing headers for buckets. Header is printed prior to listing the contents of the bucket. Args: bucket_listing_ref: BucketListingRef of type BUCKET. """ pass def PrintDir(bucket_listing_ref): """Default function for printing buckets or prefixes. Args: bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. """ text_util.print_to_fd(bucket_listing_ref.url_string) # pylint: disable=unused-argument def PrintDirSummary(num_bytes, bucket_listing_ref): """Off-by-default function for printing buckets or prefix size summaries. Args: num_bytes: Number of bytes contained in the directory. bucket_listing_ref: BucketListingRef of type BUCKET or PREFIX. """ pass def PrintDirHeader(bucket_listing_ref): """Default function for printing headers for prefixes. Header is printed prior to listing the contents of the prefix. Args: bucket_listing_ref: BucketListingRef of type PREFIX. """ text_util.print_to_fd('{}:'.format(bucket_listing_ref.url_string)) def PrintNewLine(): """Default function for printing new lines between directories.""" text_util.print_to_fd() # pylint: disable=too-many-statements def PrintFullInfoAboutObject(bucket_listing_ref, incl_acl=True): """Print full info for given object (like what displays for gsutil ls -L). Args: bucket_listing_ref: BucketListingRef being listed. Must have ref_type OBJECT and a populated root_object with the desired fields. incl_acl: True if ACL info should be output. Returns: Tuple (number of objects, object_length) Raises: Exception: if calling bug encountered. """ url_str = bucket_listing_ref.url_string storage_url = StorageUrlFromString(url_str) obj = bucket_listing_ref.root_object if (obj.metadata and S3_DELETE_MARKER_GUID in obj.metadata.additionalProperties): num_bytes = 0 num_objs = 0 url_str += '<DeleteMarker>' else: num_bytes = obj.size num_objs = 1 text_util.print_to_fd('{}:'.format(url_str)) if obj.timeCreated: text_util.print_to_fd( MakeMetadataLine('Creation time', obj.timeCreated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.updated: text_util.print_to_fd( MakeMetadataLine('Update time', obj.updated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if (obj.timeStorageClassUpdated and obj.timeStorageClassUpdated != obj.timeCreated): text_util.print_to_fd( MakeMetadataLine( 'Storage class update time', obj.timeStorageClassUpdated.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.storageClass: text_util.print_to_fd(MakeMetadataLine('Storage class', obj.storageClass)) if obj.temporaryHold: text_util.print_to_fd(MakeMetadataLine('Temporary Hold', 'Enabled')) if obj.eventBasedHold: text_util.print_to_fd(MakeMetadataLine('Event-Based Hold', 'Enabled')) if obj.retentionExpirationTime: text_util.print_to_fd( MakeMetadataLine( 'Retention Expiration', obj.retentionExpirationTime.strftime('%a, %d %b %Y %H:%M:%S GMT'))) if obj.kmsKeyName: text_util.print_to_fd(MakeMetadataLine('KMS key', obj.kmsKeyName)) if obj.cacheControl: text_util.print_to_fd(MakeMetadataLine('Cache-Control', obj.cacheControl)) if obj.contentDisposition: text_util.print_to_fd( MakeMetadataLine('Content-Disposition', obj.contentDisposition)) if obj.contentEncoding: text_util.print_to_fd( MakeMetadataLine('Content-Encoding', obj.contentEncoding)) if obj.contentLanguage: text_util.print_to_fd( MakeMetadataLine('Content-Language', obj.contentLanguage)) text_util.print_to_fd(MakeMetadataLine('Content-Length', obj.size)) text_util.print_to_fd(MakeMetadataLine('Content-Type', obj.contentType)) if obj.componentCount: text_util.print_to_fd( MakeMetadataLine('Component-Count', obj.componentCount)) if obj.customTime: text_util.print_to_fd(MakeMetadataLine('Custom-Time', obj.customTime)) if obj.timeDeleted: text_util.print_to_fd( MakeMetadataLine('Noncurrent time', obj.timeDeleted.strftime('%a, %d %b %Y %H:%M:%S GMT'))) marker_props = {} if obj.metadata and obj.metadata.additionalProperties: non_marker_props = [] for add_prop in obj.metadata.additionalProperties: if add_prop.key not in S3_MARKER_GUIDS: non_marker_props.append(add_prop) else: marker_props[add_prop.key] = add_prop.value if non_marker_props: text_util.print_to_fd(MakeMetadataLine('Metadata', '')) for ap in non_marker_props: ap_key = '{}'.format(ap.key) ap_value = '{}'.format(ap.value) meta_data_line = MakeMetadataLine(ap_key, ap_value, indent=2) text_util.print_to_fd(meta_data_line) if obj.customerEncryption: if not obj.crc32c: text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', 'encrypted')) if not obj.md5Hash: text_util.print_to_fd(MakeMetadataLine('Hash (md5)', 'encrypted')) text_util.print_to_fd( MakeMetadataLine('Encryption algorithm', obj.customerEncryption.encryptionAlgorithm)) text_util.print_to_fd( MakeMetadataLine('Encryption key SHA256', obj.customerEncryption.keySha256)) if obj.crc32c: text_util.print_to_fd(MakeMetadataLine('Hash (crc32c)', obj.crc32c)) if obj.md5Hash: text_util.print_to_fd(MakeMetadataLine('Hash (md5)', obj.md5Hash)) text_util.print_to_fd(MakeMetadataLine('ETag', obj.etag.strip('"\''))) if obj.generation: generation_str = GenerationFromUrlAndString(storage_url, obj.generation) text_util.print_to_fd(MakeMetadataLine('Generation', generation_str)) if obj.metageneration: text_util.print_to_fd(MakeMetadataLine('Metageneration', obj.metageneration)) if incl_acl: # JSON API won't return acls as part of the response unless we have # full control scope if obj.acl: text_util.print_to_fd( MakeMetadataLine('ACL', AclTranslation.JsonFromMessage(obj.acl))) elif S3_ACL_MARKER_GUID in marker_props: text_util.print_to_fd( MakeMetadataLine('ACL', marker_props[S3_ACL_MARKER_GUID])) else: # Empty ACLs are possible with Bucket Policy Only and no longer imply # ACCESS DENIED anymore. text_util.print_to_fd(MakeMetadataLine('ACL', '[]')) return (num_objs, num_bytes) def PrintObject(bucket_listing_ref): """Default printing function for objects. Args: bucket_listing_ref: BucketListingRef of type OBJECT. Returns: (num_objects, num_bytes). """ try: text_util.print_to_fd(bucket_listing_ref.url_string) except IOError as e: # Windows throws an IOError 0 here for object names containing Unicode # chars. Ignore it. if not (IS_WINDOWS and e.errno == 0): raise return (1, 0) class LsHelper(object): """Helper class for ls and du.""" def __init__(self, iterator_func, logger, print_object_func=PrintObject, print_dir_func=PrintDir, print_dir_header_func=PrintDirHeader, print_bucket_header_func=PrintBucketHeader, print_dir_summary_func=PrintDirSummary, print_newline_func=PrintNewLine, all_versions=False, should_recurse=False, exclude_patterns=None, fields=('name',), list_subdir_contents=True): """Initializes the helper class to prepare for listing. Args: iterator_func: Function for instantiating iterator. Inputs- url_string- Url string to iterate on. May include wildcards. all_versions=False- If true, iterate over all object versions. logger: Logger for outputting warnings / errors. print_object_func: Function for printing objects. print_dir_func: Function for printing buckets/prefixes. print_dir_header_func: Function for printing header line for buckets or prefixes. print_bucket_header_func: Function for printing header line for buckets or prefixes. print_dir_summary_func: Function for printing size summaries about buckets/prefixes. print_newline_func: Function for printing new lines between dirs. all_versions: If true, list all object versions. should_recurse: If true, recursively listing buckets/prefixes. exclude_patterns: Patterns to exclude when listing. fields: Fields to request from bucket listings; this should include all fields that need to be populated in objects so they can be listed. Can be set to None to retrieve all object fields. Defaults to short listing fields. list_subdir_contents: If true, return the directory and any contents, otherwise return only the directory itself. """ self._iterator_func = iterator_func self.logger = logger self._print_object_func = print_object_func self._print_dir_func = print_dir_func self._print_dir_header_func = print_dir_header_func self._print_bucket_header_func = print_bucket_header_func self._print_dir_summary_func = print_dir_summary_func self._print_newline_func = print_newline_func self.all_versions = all_versions self.should_recurse = should_recurse self.exclude_patterns = exclude_patterns self.bucket_listing_fields = fields self.list_subdir_contents = list_subdir_contents def ExpandUrlAndPrint(self, url): """Iterates over the given URL and calls print functions. Args: url: StorageUrl to iterate over. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. """ num_objects = 0 num_dirs = 0 num_bytes = 0 print_newline = False if url.IsBucket() or self.should_recurse: # IsBucket() implies a top-level listing. if url.IsBucket(): self._print_bucket_header_func(url) return self._RecurseExpandUrlAndPrint(url.url_string, print_initial_newline=False) else: # User provided a prefix or object URL, but can't tell which unless there # is a generation or we do a listing to see what matches. if url.HasGeneration(): iteration_url = url.url_string else: iteration_url = url.CreatePrefixUrl() top_level_iterator = PluralityCheckableIterator( self._iterator_func( iteration_url, all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=self.bucket_listing_fields)) plurality = top_level_iterator.HasPlurality() try: top_level_iterator.PeekException() except EncryptionException: # Detailed listing on a single object can perform a GetObjectMetadata # call, which raises if a matching encryption key isn't found. # Re-iterate without requesting encrypted fields. top_level_iterator = PluralityCheckableIterator( self._iterator_func( url.CreatePrefixUrl(wildcard_suffix=None), all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=UNENCRYPTED_FULL_LISTING_FIELDS)) plurality = top_level_iterator.HasPlurality() for blr in top_level_iterator: if self._MatchesExcludedPattern(blr): continue if blr.IsObject(): nd = 0 no, nb = self._print_object_func(blr) print_newline = True elif blr.IsPrefix(): if print_newline: self._print_newline_func() else: print_newline = True if plurality and self.list_subdir_contents: self._print_dir_header_func(blr) elif plurality and not self.list_subdir_contents: print_newline = False expansion_url_str = StorageUrlFromString( blr.url_string).CreatePrefixUrl( wildcard_suffix='' if self.list_subdir_contents else None) nd, no, nb = self._RecurseExpandUrlAndPrint(expansion_url_str) self._print_dir_summary_func(nb, blr) else: # We handle all buckets at the top level, so this should never happen. raise CommandException( 'Sub-level iterator returned a CsBucketListingRef of type Bucket') num_objects += no num_dirs += nd num_bytes += nb return num_dirs, num_objects, num_bytes def _RecurseExpandUrlAndPrint(self, url_str, print_initial_newline=True): """Iterates over the given URL string and calls print functions. Args: url_str: String describing StorageUrl to iterate over. Must be of depth one or higher. print_initial_newline: If true, print a newline before recursively expanded prefixes. Returns: (num_objects, num_bytes) total number of objects and bytes iterated. """ num_objects = 0 num_dirs = 0 num_bytes = 0 for blr in self._iterator_func( '%s' % url_str, all_versions=self.all_versions).IterAll( expand_top_level_buckets=True, bucket_listing_fields=self.bucket_listing_fields): if self._MatchesExcludedPattern(blr): continue if blr.IsObject(): nd = 0 no, nb = self._print_object_func(blr) elif blr.IsPrefix(): if self.should_recurse: if print_initial_newline: self._print_newline_func() else: print_initial_newline = True self._print_dir_header_func(blr) expansion_url_str = StorageUrlFromString( blr.url_string).CreatePrefixUrl(wildcard_suffix='') nd, no, nb = self._RecurseExpandUrlAndPrint(expansion_url_str) self._print_dir_summary_func(nb, blr) else: nd, no, nb = 1, 0, 0 self._print_dir_func(blr) else: # We handle all buckets at the top level, so this should never happen. raise CommandException( 'Sub-level iterator returned a bucketListingRef of type Bucket') num_dirs += nd num_objects += no num_bytes += nb return num_dirs, num_objects, num_bytes def _MatchesExcludedPattern(self, blr): """Checks bucket listing reference against patterns to exclude. Args: blr: BucketListingRef to check. Returns: True if reference matches a pattern and should be excluded. """ if self.exclude_patterns: tomatch = six.ensure_str(blr.url_string) for pattern in self.exclude_patterns: if fnmatch.fnmatch(tomatch, six.ensure_str(pattern)): return True return False	17.89 KB	2025-06-25 08:19
📄 metadata_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods for manipulating metadata of requests and resources.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import six from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages def AddAcceptEncodingGzipIfNeeded(headers_dict, compressed_encoding=False): if compressed_encoding: # If we send accept-encoding: gzip with a range request, the service # may respond with the whole object, which would be bad for resuming. # So only accept gzip encoding if the object we are downloading has # a gzip content encoding. # TODO: If we want to support compressive transcoding fully in the client, # condition on whether we are requesting the entire range of the object. # In this case, we can accept the first bytes of the object compressively # transcoded, but we must perform data integrity checking on bytes after # they are decompressed on-the-fly, and any connection break must be # resumed without compressive transcoding since we cannot specify an # offset. We would also need to ensure that hashes for downloaded data # from objects stored with content-encoding:gzip continue to be calculated # prior to our own on-the-fly decompression so they match the stored hashes. headers_dict['accept-encoding'] = 'gzip' def CreateCustomMetadata(entries=None, custom_metadata=None): """Creates a custom MetadataValue object. Inserts the key/value pairs in entries. Args: entries: (Dict[str, Any] or None) The dictionary containing key/value pairs to insert into metadata. Both the key and value must be able to be casted to a string type. custom_metadata (apitools_messages.Object.MetadataValue or None): A pre-existing custom metadata object to add to. If one is not provided, a new one will be constructed. Returns: An apitools_messages.Object.MetadataValue. """ if custom_metadata is None: custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) if entries is None: entries = {} for key, value in six.iteritems(entries): custom_metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=str(key), value=str(value))) return custom_metadata def GetValueFromObjectCustomMetadata(obj_metadata, search_key, default_value=None): """Filters a specific element out of an object's custom metadata. Args: obj_metadata: (apitools_messages.Object) The metadata for an object. search_key: (str) The custom metadata key to search for. default_value: (Any) The default value to use for the key if it cannot be found. Returns: (Tuple(bool, Any)) A tuple indicating if the value could be found in metadata and a value corresponding to search_key (the value at the specified key in custom metadata, or the default value if the specified key does not exist in the custom metadata). """ try: value = next((attr.value for attr in obj_metadata.metadata.additionalProperties if attr.key == search_key), None) if value is None: return False, default_value return True, value except AttributeError: return False, default_value def IsCustomMetadataHeader(header): """Returns true if header (which must be lowercase) is a custom header.""" return header.startswith('x-goog-meta-') or header.startswith('x-amz-meta-') def ObjectIsGzipEncoded(obj_metadata): """Returns true if the apitools_messages.Object has gzip content-encoding.""" return (obj_metadata.contentEncoding and obj_metadata.contentEncoding.lower().endswith('gzip'))	4.4 KB	2025-06-25 08:19
📄 parallelism_framework_util.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility classes and methods for the parallelism framework.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import errno import logging import multiprocessing import threading import traceback from gslib.utils import constants from gslib.utils import system_util from six.moves import queue as Queue # pylint: disable=g-import-not-at-top try: # This module doesn't necessarily exist on Windows. import resource _HAS_RESOURCE_MODULE = True except ImportError as e: _HAS_RESOURCE_MODULE = False # Maximum time to wait (join) on the SeekAheadThread after the ProducerThread # completes, in seconds. SEEK_AHEAD_JOIN_TIMEOUT = 60 # Timeout for puts/gets to the global status queue, in seconds. STATUS_QUEUE_OP_TIMEOUT = 5 # Maximum time to wait (join) on the UIThread after the Apply # completes, in seconds. UI_THREAD_JOIN_TIMEOUT = 60 ZERO_TASKS_TO_DO_ARGUMENT = ('There were no', 'tasks to do') # Multiprocessing manager used to coordinate across all processes. This # attribute is only present if multiprocessing is available, which can be # determined by calling CheckMultiprocessingAvailableAndInit(). global top_level_manager # pylint: disable=global-at-module-level # Cache the values from this check such that they're available to all callers # without needing to run all the checks again (some of these, such as calling # multiprocessing.Manager(), are expensive operations). _cached_multiprocessing_is_available = None _cached_multiprocessing_check_stack_trace = None _cached_multiprocessing_is_available_message = None # This must be defined at the module level for pickling across processes. MultiprocessingIsAvailableResult = collections.namedtuple( 'MultiprocessingIsAvailableResult', ['is_available', 'stack_trace']) # Explicitly set start method to 'fork' since this isn't always the default # in later versions of Python. try: multiprocessing_context = multiprocessing.get_context('fork') except (AttributeError, ValueError): multiprocessing_context = multiprocessing class AtomicDict(object): """Thread-safe (and optionally process-safe) dictionary protected by a lock. If a multiprocessing.Manager is supplied on init, the dictionary is both process and thread safe. Otherwise, it is only thread-safe. """ def __init__(self, manager=None): """Initializes the dict. Args: manager: (multiprocessing.Manager or None) Manager instance (required for cross-process safety), or none if cross-process safety is not needed. """ if manager: self.lock = manager.Lock() self.dict = manager.dict() else: self.lock = threading.Lock() self.dict = {} def __getitem__(self, key): with self.lock: return self.dict[key] def __setitem__(self, key, value): with self.lock: self.dict[key] = value # pylint: disable=invalid-name def get(self, key, default_value=None): with self.lock: return self.dict.get(key, default_value) def delete(self, key): with self.lock: del self.dict[key] def values(self): with self.lock: return self.dict.values() def Increment(self, key, inc, default_value=0): """Atomically updates the stored value associated with the given key. Performs the atomic equivalent of dict[key] = dict.get(key, default_value) + inc. Args: key: lookup key for the value of the first operand of the "+" operation. inc: Second operand of the "+" operation. default_value: Default value if there is no existing value for the key. Returns: Incremented value. """ with self.lock: val = self.dict.get(key, default_value) + inc self.dict[key] = val return val class ProcessAndThreadSafeInt(object): """This class implements a process and thread-safe integer. It is backed either by a multiprocessing Value of type 'i' or an internal threading lock. This simplifies the calling pattern for global variables that could be a Multiprocessing.Value or an integer. Without this class, callers need to write code like this: global variable_name if isinstance(variable_name, int): return variable_name else: return variable_name.value """ def __init__(self, multiprocessing_is_available): self.multiprocessing_is_available = multiprocessing_is_available if self.multiprocessing_is_available: # Lock is implicit in multiprocessing.Value self.value = multiprocessing_context.Value('i', 0) else: self.lock = threading.Lock() self.value = 0 def Reset(self, reset_value=0): if self.multiprocessing_is_available: self.value.value = reset_value else: with self.lock: self.value = reset_value def Increment(self): if self.multiprocessing_is_available: self.value.value += 1 else: with self.lock: self.value += 1 def Decrement(self): if self.multiprocessing_is_available: self.value.value -= 1 else: with self.lock: self.value -= 1 def GetValue(self): if self.multiprocessing_is_available: return self.value.value else: with self.lock: return self.value def _IncreaseSoftLimitForResource(resource_name, fallback_value): """Sets a new soft limit for the maximum number of open files. The soft limit is used for this process (and its children), but the hard limit is set by the system and cannot be exceeded. We will first try to set the soft limit to the hard limit's value; if that fails, we will try to set the soft limit to the fallback_value iff this would increase the soft limit. Args: resource_name: Name of the resource to increase the soft limit for. fallback_value: Fallback value to be used if we couldn't set the soft value to the hard value (e.g., if the hard value is "unlimited"). Returns: Current soft limit for the resource (after any changes we were able to make), or -1 if the resource doesn't exist. """ # Get the value of the resource. try: (soft_limit, hard_limit) = resource.getrlimit(resource_name) except (resource.error, ValueError): # The resource wasn't present, so we can't do anything here. return -1 # Try to set the value of the soft limit to the value of the hard limit. if hard_limit > soft_limit: # Some OS's report 0 for "unlimited". try: resource.setrlimit(resource_name, (hard_limit, hard_limit)) return hard_limit except (resource.error, ValueError): # We'll ignore this and try the fallback value. pass # Try to set the value of the soft limit to the fallback value. if soft_limit < fallback_value: try: resource.setrlimit(resource_name, (fallback_value, hard_limit)) return fallback_value except (resource.error, ValueError): # We couldn't change the soft limit, so just report the current # value of the soft limit. return soft_limit else: return soft_limit def ShouldProhibitMultiprocessing(): """Determines if the OS doesn't support multiprocessing. There are two cases we currently know about: - Multiple processes are not supported on Windows. - If an error is encountered while using multiple processes on Alpine Linux gsutil hangs. For this case it's possible we could do more work to find the root cause but after a fruitless initial attempt we decided instead to fall back on multi-threading w/o multiprocesing. Returns: (bool indicator if multiprocessing should be prohibited, OS name) """ if system_util.IS_WINDOWS: # Issues have been observed while trying to use multi-processing in Windows return (True, 'Windows') if system_util.IS_OSX: # macOS does not contain /etc/os-release, used in this method. This # shortcuts raising an exception and returning 'Unknown' as an OS. return (False, 'macOS') try: with open('/etc/os-release', 'r') as f: # look for line that contains 'NAME=' both PRETTY_NAME and NAME should # be acceptable to try to find if alpine linux is being used. for line in f.read().splitlines(): if 'NAME=' in line: os_name = line.split('=')[1].strip('"') return ('alpine linux' in os_name.lower(), os_name) # Unable to determine OS. NAME line not found in /etc/os-release file. return (False, 'Unknown') except IOError as e: if e.errno == errno.ENOENT: logging.debug('Unable to open /etc/os-release to determine whether OS ' 'supports multiprocessing: errno=%d, message=%s' % (e.errno, str(e))) return (False, 'Unknown') else: raise except Exception as exc: logging.debug('Something went wrong while trying to determine ' 'multiprocessing capabilities.\nMessage: {0}'.format( str(exc))) return (False, 'Unknown') def CheckMultiprocessingAvailableAndInit(logger=None): """Checks if multiprocessing is available, and if so performs initialization. There are some environments in which there is no way to use multiprocessing logic that's built into Python (e.g., if /dev/shm is not available, then we can't create semaphores). This simply tries out a few things that will be needed to make sure the environment can support the pieces of the multiprocessing module that we need. See gslib.command.InitializeMultiprocessingVariables for an explanation of why this is necessary. Args: logger: (logging.Logger) Logger to use for debug output. Returns: (MultiprocessingIsAvailableResult) A namedtuple with the following attrs: - multiprocessing_is_available: True iff the multiprocessing module is available for use. - stack_trace: The stack trace generated by the call we tried that failed. """ # pylint: disable=global-variable-undefined global _cached_multiprocessing_is_available global _cached_multiprocessing_check_stack_trace global _cached_multiprocessing_is_available_message if _cached_multiprocessing_is_available is not None: if logger: logger.debug(_cached_multiprocessing_check_stack_trace) logger.warn(_cached_multiprocessing_is_available_message) return MultiprocessingIsAvailableResult( is_available=_cached_multiprocessing_is_available, stack_trace=_cached_multiprocessing_check_stack_trace) should_prohibit_multiprocessing, os_name = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing: message = """ Multiple processes are not supported on %s. Operations requesting parallelism will be executed with multiple threads in a single process only. """ % os_name if logger: logger.warn(message) return MultiprocessingIsAvailableResult(is_available=False, stack_trace=None) stack_trace = None multiprocessing_is_available = True message = """ You have requested multiple processes for an operation, but the required functionality of Python\'s multiprocessing module is not available. Operations requesting parallelism will be executed with multiple threads in a single process only. """ try: # Fails if /dev/shm (or some equivalent thereof) is not available for use # (e.g., there's no implementation, or we can't write to it, etc.). try: multiprocessing_context.Value('i', 0) except: message += """ Please ensure that you have write access to both /dev/shm and /run/shm. """ raise # We'll handle this in one place below. global top_level_manager # pylint: disable=global-variable-undefined top_level_manager = multiprocessing_context.Manager() # Check that the max number of open files is reasonable. Always check this # after we're sure that the basic multiprocessing functionality is # available, since this won't matter unless that's true. limit = -1 if _HAS_RESOURCE_MODULE: # Try to set this with both resource names - RLIMIT_NOFILE for most Unix # platforms, and RLIMIT_OFILE for BSD. Ignore AttributeError because the # "resource" module is not guaranteed to know about these names. try: limit = max( limit, _IncreaseSoftLimitForResource( resource.RLIMIT_NOFILE, constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT)) except AttributeError: pass try: limit = max( limit, _IncreaseSoftLimitForResource( resource.RLIMIT_OFILE, constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT)) except AttributeError: pass if limit < constants.MIN_ACCEPTABLE_OPEN_FILES_LIMIT: message += (""" Your max number of open files, %s, is too low to allow safe multiprocessing. On Linux you can fix this by adding something like "ulimit -n 10000" to your ~/.bashrc or equivalent file and opening a new terminal. On macOS, you may also need to run a command like this once (in addition to the above instructions), which might require a restart of your system to take effect: launchctl limit maxfiles 10000 Alternatively, edit /etc/launchd.conf with something like: limit maxfiles 10000 10000 """ % limit) raise Exception('Max number of open files, %s, is too low.' % limit) except: # pylint: disable=bare-except stack_trace = traceback.format_exc() multiprocessing_is_available = False if logger is not None: logger.debug(stack_trace) logger.warn(message) # Set the cached values so that we never need to do this check again. _cached_multiprocessing_is_available = multiprocessing_is_available _cached_multiprocessing_check_stack_trace = stack_trace _cached_multiprocessing_is_available_message = message return MultiprocessingIsAvailableResult( is_available=_cached_multiprocessing_is_available, stack_trace=_cached_multiprocessing_check_stack_trace) def CreateLock(): """Returns either a multiprocessing lock or a threading lock. Use Multiprocessing lock iff we have access to the parts of the multiprocessing module that are necessary to enable parallelism in operations. Returns: Multiprocessing or threading lock. """ if CheckMultiprocessingAvailableAndInit().is_available: return top_level_manager.Lock() else: return threading.Lock() # Pylint gets confused by the mixed lower and upper-case method names in # AtomicDict. # pylint: disable=invalid-name def PutToQueueWithTimeout(queue, msg, timeout=STATUS_QUEUE_OP_TIMEOUT): """Puts an item to the status queue. If the queue is full, this function will timeout periodically and repeat until success. This avoids deadlock during shutdown by never making a fully blocking call to the queue, since Python signal handlers cannot execute in between instructions of the Python interpreter (see https://docs.python.org/2/library/signal.html for details). Args: queue: Queue class (typically the global status queue) msg: message to post to the queue. timeout: (optional) amount of time to wait before repeating put request. """ put_success = False while not put_success: try: queue.put(msg, timeout=timeout) put_success = True except Queue.Full: pass # pylint: enable=invalid-name	15.72 KB	2025-06-25 08:19
📄 posix_util.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper file for POSIX methods.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from calendar import timegm import getpass import logging import os import re import time import six from gslib.exception import CommandException from gslib.tz_utc import UTC from gslib.utils.metadata_util import CreateCustomMetadata from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata from gslib.utils.system_util import IS_WINDOWS from gslib.utils.unit_util import SECONDS_PER_DAY # pylint: disable=g-import-not-at-top if not IS_WINDOWS: import grp import pwd if six.PY3: long = int # Metadata attribute names for POSIX attributes. ATIME_ATTR = 'goog-reserved-file-atime' GID_ATTR = 'goog-reserved-posix-gid' MODE_ATTR = 'goog-reserved-posix-mode' MTIME_ATTR = 'goog-reserved-file-mtime' UID_ATTR = 'goog-reserved-posix-uid' # NA_TIME is a long value that takes the place of any invalid mtime. NA_TIME = -1 # NA_ID is a value that takes the place of any invalid POSIX UID or GID. NA_ID = -1 # NA_MODE is an octal value that takes the place of any invalid POSIX file mode. NA_MODE = -0o1 MODE_REGEX = re.compile('^[0-7]{3}$') # POSIX Permissions Bit-masks # User. U_R = 0o400 U_W = 0o200 U_X = 0o100 # Group. G_R = 0o040 G_W = 0o020 G_X = 0o010 # Other. O_R = 0o004 O_W = 0o002 O_X = 0o001 # The permissions newly created files get by default. SYSTEM_POSIX_MODE = None # A list of group IDs that the current user is a member of. USER_GROUPS = set() class POSIXAttributes(object): """Class to hold POSIX attributes for a file/object.""" def __init__(self, atime=NA_TIME, mtime=NA_TIME, uid=NA_ID, gid=NA_ID, mode=None): """Constructor for POSIXAttributes class which holds relevant data. Args: atime: The access time of the file/object. mtime: The modification time of the file/object. uid: The user ID that owns the file. gid: The group ID that the user is in. mode: An instance of POSIXMode. """ self.atime = atime self.mtime = mtime self.uid = uid self.gid = gid self.mode = POSIXMode(mode if mode else NA_MODE) class POSIXMode(object): def __init__(self, permissions): self.permissions = permissions def ConvertModeToBase8(mode): """Converts a base-10 mode integer from os.stat to base-8.""" # Strip out unnecessary bits in the mode. Mode is given as a base-10 # integer. It must be converted to base-8. return int(oct(mode)[-3:]) def DeserializeFileAttributesFromObjectMetadata(obj_metadata, url_str): """Parses the POSIX attributes from the supplied metadata. Args: obj_metadata: The metadata for an object. url_str: File/object path that provides context if a warning is thrown. Returns: A POSIXAttribute object with the retrieved values or a default value for any attribute that could not be found. """ posix_attrs = POSIXAttributes() # Parse atime. found, atime = GetValueFromObjectCustomMetadata(obj_metadata, ATIME_ATTR, NA_TIME) try: atime = long(atime) if found and atime <= NA_TIME: WarnNegativeAttribute('atime', url_str) atime = NA_TIME elif atime > long(time.time()) + SECONDS_PER_DAY: WarnFutureTimestamp('atime', url_str) atime = NA_TIME except ValueError: WarnInvalidValue('atime', url_str) atime = NA_TIME posix_attrs.atime = atime # Parse gid. DeserializeIDAttribute(obj_metadata, GID_ATTR, url_str, posix_attrs) # Parse uid. DeserializeIDAttribute(obj_metadata, UID_ATTR, url_str, posix_attrs) found, mode = GetValueFromObjectCustomMetadata(obj_metadata, MODE_ATTR, NA_MODE) if found and MODE_REGEX.match(mode): try: # Parse mode into a 3-digit base-8 number. posix_attrs.mode = POSIXMode(int(mode)) except ValueError: WarnInvalidValue('mode', url_str) return posix_attrs def SerializeFileAttributesToObjectMetadata(posix_attrs, custom_metadata, preserve_posix=False): """Takes a POSIXAttributes object and serializes it into custom metadata. Args: posix_attrs: A POSIXAttributes object. custom_metadata: A custom metadata object to serialize values into. preserve_posix: Whether or not to preserve POSIX attributes other than mtime. """ # mtime will always be needed in the object metadata for rsync. if posix_attrs.mtime != NA_TIME: CreateCustomMetadata(entries={MTIME_ATTR: posix_attrs.mtime}, custom_metadata=custom_metadata) # Only add other POSIX attributes if the preserve_posix flag is set. if preserve_posix: if posix_attrs.atime != NA_TIME: CreateCustomMetadata(entries={ATIME_ATTR: posix_attrs.atime}, custom_metadata=custom_metadata) if posix_attrs.uid != NA_ID: CreateCustomMetadata(entries={UID_ATTR: posix_attrs.uid}, custom_metadata=custom_metadata) if posix_attrs.gid != NA_ID: CreateCustomMetadata(entries={GID_ATTR: posix_attrs.gid}, custom_metadata=custom_metadata) if posix_attrs.mode.permissions != NA_MODE: CreateCustomMetadata(entries={MODE_ATTR: posix_attrs.mode.permissions}, custom_metadata=custom_metadata) def DeserializeIDAttribute(obj_metadata, attr, url_str, posix_attrs): """Parses the POSIX attributes from the supplied metadata into posix_attrs. Args: obj_metadata: The metadata for an object. attr: Either GID_ATTR or UID_ATTR. url_str: File/object path that provides context if a warning is thrown. posix_attrs: POSIXAttribute object. """ attr_name = attr.split('-')[-1] found, val = GetValueFromObjectCustomMetadata(obj_metadata, attr, NA_ID) try: val = int(val) if found and val <= NA_ID: WarnNegativeAttribute(attr_name, url_str) val = NA_ID except ValueError: WarnInvalidValue(attr_name, url_str) val = NA_ID setattr(posix_attrs, attr_name, val) def NeedsPOSIXAttributeUpdate(src_atime, dst_atime, src_mtime, dst_mtime, src_uid, dst_uid, src_gid, dst_gid, src_mode, dst_mode): """Checks whether an update for any POSIX attribute is needed. Args: src_atime: The source access time. dst_atime: The destination access time. src_mtime: The source modification time. dst_mtime: The destination modification time. src_uid: The source user ID. dst_uid: The destination user ID. src_gid: The source group ID. dst_gid: The destination group ID. src_mode: The source mode. dst_mode: The destination mode. Returns: A tuple containing a POSIXAttribute object and a boolean for whether an update was needed. """ posix_attrs = POSIXAttributes() has_src_atime = src_atime > NA_TIME has_dst_atime = dst_atime > NA_TIME has_src_mtime = src_mtime > NA_TIME has_dst_mtime = dst_mtime > NA_TIME has_src_uid = src_uid > NA_ID has_dst_uid = dst_uid > NA_ID has_src_gid = src_gid > NA_ID has_dst_gid = dst_gid > NA_ID has_src_mode = src_mode > NA_MODE has_dst_mode = dst_mode > NA_MODE if has_src_atime and not has_dst_atime: posix_attrs.atime = src_atime if has_src_mtime and not has_dst_mtime: posix_attrs.mtime = src_mtime if has_src_uid and not has_dst_uid: posix_attrs.uid = src_uid if has_src_gid and not has_dst_gid: posix_attrs.gid = src_gid if has_src_mode and not has_dst_mode: posix_attrs.mode.permissions = src_mode return posix_attrs, ((has_src_atime and not has_dst_atime) or (has_src_mtime and not has_dst_mtime) or (has_src_uid and not has_dst_uid) or (has_src_gid and not has_dst_gid) or (has_src_mode and not has_dst_mode)) def ValidatePOSIXMode(mode): """Validates whether the mode is valid. In order for the mode to be valid either the user, group, or other byte must be >= 4. Args: mode: The mode as a 3-digit, base-8 integer. Returns: True/False """ return MODE_REGEX.match(oct(mode)[-3:]) and (mode & U_R or mode & G_R or mode & O_R) def ValidateFilePermissionAccess(url_str, uid=NA_ID, gid=NA_ID, mode=NA_MODE): """Validates that the user has file access if uid, gid, and mode are applied. Args: url_str: The path to the object for which this is validating. uid: A POSIX user ID. gid: A POSIX group ID. mode: A 3-digit, number representing POSIX permissions, must be in base-8. Returns: A (bool, str) tuple, True if and only if it's safe to copy the file, and a string containing details for the error. """ # Windows doesn't use the POSIX system for file permissions, so all files will # validate. if IS_WINDOWS: return True, '' uid_present = uid > NA_ID gid_present = int(gid) > NA_ID mode_present = mode > NA_MODE # No need to perform validation if Posix attrs are not being preserved. if not (uid_present or gid_present or mode_present): return True, '' # The root user on non-Windows systems can access files regardless of their # permissions. if os.geteuid() == 0: return True, '' mode_valid = ValidatePOSIXMode(int(str(mode), 8)) if mode_present: if not mode_valid: return False, 'Mode for %s won\'t allow read access.' % url_str else: # Calculate the default mode if the mode doesn't exist. # Convert mode to a 3-digit, base-8 integer. mode = int(SYSTEM_POSIX_MODE) if uid_present: try: pwd.getpwuid(uid) except (KeyError, OverflowError): return (False, 'UID for %s doesn\'t exist on current system. uid: %d' % (url_str, uid)) if gid_present: try: grp.getgrgid(gid) except (KeyError, OverflowError): return (False, 'GID for %s doesn\'t exist on current system. gid: %d' % (url_str, gid)) # uid at this point must exist, but isn't necessarily the current user. # Likewise, gid must also exist at this point. uid_is_current_user = uid == os.getuid() # By this point uid and gid must exist on the system. However, the uid might # not match the current user's or the current user might not be a member of # the group identified by gid. In this case, the 'other' byte of the # permissions could provide sufficient access. mode = int(str(mode), 8) # Check that if the uid is not present and the gid and mode are, so that we # won't orphan the file. For example if the mode is set to 007, we can orphan # the file because the uid would default to the current user's ID and if the # current user wouldn't have read access or better, the file will be orphaned # even though they might otherwise have access through the gid or other bytes. if not uid_present and gid_present and mode_present and not bool(mode & U_R): return (False, 'Insufficient access with uid/gid/mode for %s, gid: %d, ' 'mode: %s' % (url_str, gid, oct(mode)[-3:])) if uid_is_current_user: valid = bool(mode & U_R) return (valid, '' if valid else 'Insufficient access with uid/gid/mode for %s, uid: %d, ' 'mode: %s' % (url_str, uid, oct(mode)[-3:])) elif int(gid) in USER_GROUPS: valid = bool(mode & G_R) return (valid, '' if valid else 'Insufficient access with uid/gid/mode for %s, gid: %d, ' 'mode: %s' % (url_str, gid, oct(mode)[-3:])) elif mode & O_R: return True, '' elif not uid_present and not gid_present and mode_valid: return True, '' return False, 'There was a problem validating %s.' % url_str def ParseAndSetPOSIXAttributes(path, obj_metadata, is_rsync=False, preserve_posix=False): """Parses POSIX attributes from obj_metadata and sets them. Attributes will only be set if they exist in custom metadata. This function should only be called after ValidateFilePermissionAccess has been called for the specific file/object so as not to orphan files. Args: path: The local filesystem path for the file. Valid metadata attributes will be set for the file located at path, some attributes will only be set if preserve_posix is set to True. obj_metadata: The metadata for the file/object. is_rsync: Whether or not the caller is the rsync command. Used to determine if timeCreated should be used. preserve_posix: Whether or not all POSIX attributes should be set. """ if obj_metadata is None: # This exception is meant for debugging purposes as it should never be # thrown unless there are unexpected code changes. raise CommandException('obj_metadata cannot be None for %s' % path) try: found_at, atime = GetValueFromObjectCustomMetadata(obj_metadata, ATIME_ATTR, default_value=NA_TIME) found_mt, mtime = GetValueFromObjectCustomMetadata(obj_metadata, MTIME_ATTR, default_value=NA_TIME) found_uid, uid = GetValueFromObjectCustomMetadata(obj_metadata, UID_ATTR, default_value=NA_ID) found_gid, gid = GetValueFromObjectCustomMetadata(obj_metadata, GID_ATTR, default_value=NA_ID) found_mode, mode = GetValueFromObjectCustomMetadata(obj_metadata, MODE_ATTR, default_value=NA_MODE) if found_mt: mtime = long(mtime) if not preserve_posix: atime_tmp = os.stat(path).st_atime os.utime(path, (atime_tmp, mtime)) return elif is_rsync: mtime = ConvertDatetimeToPOSIX(obj_metadata.timeCreated) os.utime(path, (mtime, mtime)) if not preserve_posix: return if found_at: atime = long(atime) if atime > NA_TIME and mtime > NA_TIME: # Set both atime and mtime. os.utime(path, (atime, mtime)) elif atime > NA_TIME and mtime <= NA_TIME: # atime is valid but mtime isn't. mtime_tmp = os.stat(path).st_mtime os.utime(path, (atime, mtime_tmp)) elif atime <= NA_TIME and mtime > NA_TIME: # mtime is valid but atime isn't atime_tmp = os.stat(path).st_atime os.utime(path, (atime_tmp, mtime)) if IS_WINDOWS: # Windows doesn't use POSIX uid/gid/mode unlike other systems. So there's # no point continuing. return # Only root can change ownership. if found_uid and os.geteuid() == 0: uid = int(uid) else: uid = NA_ID if found_gid: gid = int(gid) if uid > NA_ID and gid > NA_ID: # Set both uid and gid. os.chown(path, uid, gid) elif uid > NA_ID and gid <= NA_ID: # uid is valid but gid isn't. os.chown(path, uid, -1) elif uid <= NA_ID and gid > NA_ID: # gid is valid but uid isn't. os.chown(path, -1, gid) if found_mode: mode = int(str(mode), 8) os.chmod(path, mode) except ValueError: raise CommandException('Check POSIX attribute values for %s' % obj_metadata.name) def WarnNegativeAttribute(attr_name, url_str): """Logs if an attribute has a negative value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn('%s has a negative %s in its metadata', url_str, attr_name) def WarnInvalidValue(attr_name, url_str): """Logs if an attribute has an invalid value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn('%s has an invalid %s in its metadata', url_str, attr_name) def WarnFutureTimestamp(attr_name, url_str): """Logs if an attribute has an invalid value. Args: attr_name: The name of the attribute to log. url_str: The path of the file for context. """ logging.getLogger().warn( '%s has an %s more than 1 day from current system' ' time', url_str, attr_name) def ConvertDatetimeToPOSIX(dt): """Converts a datetime object to UTC and formats as POSIX. Sanitize the timestamp returned in dt, and put it in UTC format. For more information see the UTC class. Args: dt: A Python datetime object. Returns: A POSIX timestamp according to UTC. """ return long(timegm(dt.replace(tzinfo=UTC()).timetuple())) def InitializeDefaultMode(): """Records the default POSIX mode using os.umask.""" global SYSTEM_POSIX_MODE if IS_WINDOWS: # os.umask returns 0 on Windows. Below math works out to 666. SYSTEM_POSIX_MODE = '666' return # umask returns the permissions that should not be granted, so they must be # subtracted from the maximum set of permissions. max_permissions = 0o777 current_umask = os.umask(0o177) os.umask(current_umask) mode = max_permissions - current_umask # Files are not given execute privileges by default. Therefore we need to # subtract one from every odd permissions value. This is done via a bitmask. SYSTEM_POSIX_MODE = oct(mode & 0o666)[-3:] def InitializeUserGroups(): """Initializes the set of groups that the user is in. Should only be called if the flag for preserving POSIX attributes is set. """ global USER_GROUPS if IS_WINDOWS: return user_id = os.getuid() user_name = pwd.getpwuid(user_id).pw_name USER_GROUPS = set( # Primary group [pwd.getpwuid(user_id).pw_gid] + # Secondary groups [g.gr_gid for g in grp.getgrall() if user_name in g.gr_mem]) def InitializePreservePosixData(): """Initializes POSIX data. Run once at the beginning of a copy.""" InitializeDefaultMode() InitializeUserGroups()	18.66 KB	2025-06-25 08:19
📄 retention_util.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of Retention Policy configuration command for buckets.""" from __future__ import absolute_import from six.moves import input from decimal import Decimal import re from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages SECONDS_IN_DAY = 24 * 60 * 60 SECONDS_IN_MONTH = 31 * SECONDS_IN_DAY SECONDS_IN_YEAR = int(365.25 * SECONDS_IN_DAY) _LOCK_PROMPT = ( 'This will PERMANENTLY set the Retention Policy on gs://{} to:\n\n' '{}\n\nThis setting cannot be reverted! Continue?') # Regex to match retention period in years. _RETENTION_IN_YEARS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)y$')) # Regex to match retention period in months. _RETENTION_IN_MONTHS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)m$')) # Regex to match retention period in days. _RETENTION_IN_DAYS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)d$')) # Regex to match retention period in seconds. _RETENTION_IN_SECONDS = LazyWrapper(lambda: re.compile(r'(?P<number>\d+)s$')) def _ConfirmWithUserPrompt(question, default_response): """Prompts user to confirm an action with yes or no response. Args: question: Yes/No question to be used for the prompt. default_response: Default response to the question: True, False Returns: Returns the rough equivalent duration in seconds. """ prompt = '' if default_response: prompt = '%s [%s\|%s]: ' % (question, 'Y', 'n') else: prompt = '%s [%s\|%s]: ' % (question, 'y', 'N') while True: response = input(prompt).lower() if not response: return default_response if response not in ['y', 'yes', 'n', 'no']: print('\tPlease respond with \'yes\'/\'y\' or \'no\'/\'n\'.') continue if response == 'yes' or response == 'y': return True if response == 'no' or response == 'n': return False def _RetentionPeriodToString(retention_period): """Converts Retention Period to Human readable format. Args: retention_period: Retention duration in seconds (integer value). Returns: Returns a string representing retention duration in human readable format. """ # TODO: add link to public documentation regarding conversion rates. period = Decimal(retention_period) duration_str = None if period // SECONDS_IN_YEAR == period / SECONDS_IN_YEAR: duration_str = '{} Year(s)'.format(period // SECONDS_IN_YEAR) elif period // SECONDS_IN_MONTH == period / SECONDS_IN_MONTH: duration_str = '{} Month(s)'.format(period // SECONDS_IN_MONTH) elif period // SECONDS_IN_DAY == period / SECONDS_IN_DAY: duration_str = '{} Day(s)'.format(period // SECONDS_IN_DAY) elif period > SECONDS_IN_DAY: duration_str = '{} Seconds (~{} Day(s))'.format(retention_period, period // SECONDS_IN_DAY) else: duration_str = '{} Second(s)'.format(retention_period) return (' Duration: {}').format(duration_str) def RetentionPolicyToString(retention_policy, bucket_url): """Converts Retention Policy to Human readable format.""" retention_policy_str = '' if retention_policy and retention_policy.retentionPeriod: locked_string = '(LOCKED)' if retention_policy.isLocked else '(UNLOCKED)' retention_period = _RetentionPeriodToString( retention_policy.retentionPeriod) retention_effective_time = ' Effective Time: {}'.format( retention_policy.effectiveTime.strftime('%a, %d %b %Y %H:%M:%S GMT')) retention_policy_str = ' Retention Policy {}:\n{}\n{}'.format( locked_string, retention_period, retention_effective_time) else: retention_policy_str = '{} has no Retention Policy.'.format(bucket_url) return retention_policy_str def ConfirmLockRequest(bucket_url, retention_policy): retention_policy = RetentionPolicyToString(retention_policy, bucket_url) lock_prompt = _LOCK_PROMPT.format(bucket_url, retention_policy) return _ConfirmWithUserPrompt(lock_prompt, False) def UpdateObjectMetadataExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(e) cls.everything_set_okay = False def SetTempHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Setting Temporary Hold on %s...' object_metadata_update = apitools_messages.Object(temporaryHold=True) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def ReleaseTempHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Releasing Temporary Hold on %s...' object_metadata_update = apitools_messages.Object(temporaryHold=False) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def SetEventHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Setting Event-Based Hold on %s...' object_metadata_update = apitools_messages.Object(eventBasedHold=True) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def ReleaseEventHoldFuncWrapper(cls, name_expansion_result, thread_state=None): log_template = 'Releasing Event-Based Hold on %s...' object_metadata_update = apitools_messages.Object(eventBasedHold=False) cls.ObjectUpdateMetadataFunc(object_metadata_update, log_template, name_expansion_result, thread_state=thread_state) def DaysToSeconds(days): """Converts duration specified in days to equivalent seconds. Args: days: Retention duration in number of days. Returns: Returns the equivalent duration in seconds. """ return days * SECONDS_IN_DAY def MonthsToSeconds(months): """Converts duration specified in months to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: months: Retention duration in number of months. Returns: Returns the rough equivalent duration in seconds. """ return months * SECONDS_IN_MONTH def YearsToSeconds(years): """Converts duration specified in years to equivalent seconds. GCS bucket lock API uses following duration equivalencies to convert durations specified in terms of months or years to seconds: - A month is considered to be 31 days or 2,678,400 seconds. - A year is considered to be 365.25 days or 31,557,600 seconds. Args: years: Retention duration in number of years. Returns: Returns the rough equivalent duration in seconds. """ return years * SECONDS_IN_YEAR def RetentionInYearsMatch(years): """Test whether the string matches retention in years pattern. Args: years: string to match for retention specified in years format. Returns: Returns a match object if the string matches the retention in years pattern. The match object will contain a 'number' group for the duration in number of years. Otherwise, None is returned. """ return _RETENTION_IN_YEARS().match(years) def RetentionInMonthsMatch(months): """Test whether the string matches retention in months pattern. Args: months: string to match for retention specified in months format. Returns: Returns a match object if the string matches the retention in months pattern. The match object will contain a 'number' group for the duration in number of months. Otherwise, None is returned. """ return _RETENTION_IN_MONTHS().match(months) def RetentionInDaysMatch(days): """Test whether the string matches retention in days pattern. Args: days: string to match for retention specified in days format. Returns: Returns a match object if the string matches the retention in days pattern. The match object will contain a 'number' group for the duration in number of days. Otherwise, None is returned. """ return _RETENTION_IN_DAYS().match(days) def RetentionInSecondsMatch(seconds): """Test whether the string matches retention in seconds pattern. Args: seconds: string to match for retention specified in seconds format. Returns: Returns a match object if the string matches the retention in seconds pattern. The match object will contain a 'number' group for the duration in number of seconds. Otherwise, None is returned. """ return _RETENTION_IN_SECONDS().match(seconds) def RetentionInSeconds(pattern): """Converts a retention period string pattern to equivalent seconds. Args: pattern: a string pattern that represents a retention period. Returns: Returns the retention period in seconds. If the pattern does not match """ seconds = None year_match = RetentionInYearsMatch(pattern) month_match = RetentionInMonthsMatch(pattern) day_match = RetentionInDaysMatch(pattern) second_match = RetentionInSecondsMatch(pattern) if year_match: seconds = YearsToSeconds(int(year_match.group('number'))) elif month_match: seconds = MonthsToSeconds(int(month_match.group('number'))) elif day_match: seconds = DaysToSeconds(int(day_match.group('number'))) elif second_match: seconds = int(second_match.group('number')) else: raise CommandException('Incorrect retention period specified. ' 'Please use one of the following formats ' 'to specify the retention period : ' '<number>y, <number>m, <number>d, <number>s.') return seconds	10.47 KB	2025-06-25 08:19
📄 retry_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for handling request retries.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import time from apitools.base.py import http_wrapper from gslib import thread_message from gslib.utils import constants from retry_decorator import retry_decorator Retry = retry_decorator.retry # pylint: disable=invalid-name def LogAndHandleRetries(is_data_transfer=False, status_queue=None): """Higher-order function allowing retry handler to access global status queue. Args: is_data_transfer: If True, disable retries in apitools. status_queue: The global status queue. Returns: A retry function for retryable errors in apitools. """ def WarnAfterManyRetriesHandler(retry_args): """Exception handler for http failures in apitools. If the user has had to wait several seconds since their first request, print a progress message to the terminal to let them know we're still retrying, then perform the default retry logic and post a gslib.thread_message.RetryableErrorMessage to the global status queue. Args: retry_args: An apitools ExceptionRetryArgs tuple. """ if (retry_args.total_wait_sec is not None and retry_args.total_wait_sec >= constants.LONG_RETRY_WARN_SEC): logging.info('Retrying request, attempt #%d...', retry_args.num_retries) if status_queue: status_queue.put( thread_message.RetryableErrorMessage( retry_args.exc, time.time(), num_retries=retry_args.num_retries, total_wait_sec=retry_args.total_wait_sec)) http_wrapper.HandleExceptionsAndRebuildHttpConnections(retry_args) def RetriesInDataTransferHandler(retry_args): """Exception handler that disables retries in apitools data transfers. Post a gslib.thread_message.RetryableErrorMessage to the global status queue. We handle the actual retries within the download and upload functions. Args: retry_args: An apitools ExceptionRetryArgs tuple. """ if status_queue: status_queue.put( thread_message.RetryableErrorMessage( retry_args.exc, time.time(), num_retries=retry_args.num_retries, total_wait_sec=retry_args.total_wait_sec)) http_wrapper.RethrowExceptionHandler(retry_args) if is_data_transfer: return RetriesInDataTransferHandler return WarnAfterManyRetriesHandler	3.11 KB	2025-06-25 08:19
📄 rsync_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for rsync functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class DiffAction(object): """Enum class representing possible actions to take for an rsync diff.""" COPY = 'copy' REMOVE = 'remove' MTIME_SRC_TO_DST = 'mtime_src_to_dst' POSIX_SRC_TO_DST = 'posix_src_to_dst' class RsyncDiffToApply(object): """Class that encapsulates info needed to apply diff for one object.""" def __init__(self, src_url_str, dst_url_str, src_posix_attrs, diff_action, copy_size): """Constructor. Args: src_url_str: (str or None) The source URL string, or None if diff_action is REMOVE. dst_url_str: (str) The destination URL string. src_posix_attrs: (posix_util.POSIXAttributes) The source POSIXAttributes. diff_action: (DiffAction) DiffAction to be applied. copy_size: (int or None) The amount of bytes to copy, or None if diff_action is REMOVE. """ self.src_url_str = src_url_str self.dst_url_str = dst_url_str self.src_posix_attrs = src_posix_attrs self.diff_action = diff_action self.copy_size = copy_size	1.84 KB	2025-06-25 08:19
📄 shim_util.py # -- coding: utf-8 -- # Copyright 2021 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper for shim used to translate gsutil command to gcloud storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import enum import os import re import subprocess from boto import config from gslib import exception from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import system_util class HIDDEN_SHIM_MODE(enum.Enum): NO_FALLBACK = 'no_fallback' DRY_RUN = 'dry_run' NONE = 'none' class RepeatFlagType(enum.Enum): LIST = 0 DICT = 1 DECRYPTION_KEY_REGEX = re.compile(r'^decryption_key([1-9]$\|[1-9][0-9]$\|100$)') # Required for headers translation and boto config translation. COMMANDS_SUPPORTING_ALL_HEADERS = frozenset(['cp', 'mv', 'rsync', 'setmeta']) ENCRYPTION_SUPPORTED_COMMANDS = COMMANDS_SUPPORTING_ALL_HEADERS \| frozenset( ['ls', 'rewrite', 'stat', 'cat', 'compose']) PRECONDITONS_ONLY_SUPPORTED_COMMANDS = frozenset( ['compose', 'rewrite', 'rm', 'retention']) DATA_TRANSFER_HEADERS = frozenset([ 'cache-control', 'content-disposition', 'content-encoding', 'content-md5', 'content-language', 'content-type', 'custom-time', ]) PRECONDITIONS_HEADERS = frozenset([ 'x-goog-generation-match', 'x-goog-if-generation-match', 'x-goog-metageneration-match', 'x-goog-if-metageneration-match' ]) # The format for _BOTO_CONFIG_MAP is as follows: # { # 'Boto section name': { # 'boto field name': 'correspnding env variable name in Cloud SDK' # } # } _BOTO_CONFIG_MAP = { 'Credentials': { 'aws_access_key_id': 'AWS_ACCESS_KEY_ID', 'aws_secret_access_key': 'AWS_SECRET_ACCESS_KEY', 'gs_access_key_id': 'CLOUDSDK_STORAGE_GS_XML_ACCESS_KEY_ID', 'gs_secret_access_key': 'CLOUDSDK_STORAGE_GS_XML_SECRET_ACCESS_KEY', 'use_client_certificate': 'CLOUDSDK_CONTEXT_AWARE_USE_CLIENT_CERTIFICATE', }, 'Boto': { 'proxy': 'CLOUDSDK_PROXY_ADDRESS', 'proxy_type': 'CLOUDSDK_PROXY_TYPE', 'proxy_port': 'CLOUDSDK_PROXY_PORT', 'proxy_user': 'CLOUDSDK_PROXY_USERNAME', 'proxy_pass': 'CLOUDSDK_PROXY_PASSWORD', 'proxy_rdns': 'CLOUDSDK_PROXY_RDNS', 'http_socket_timeout': 'CLOUDSDK_CORE_HTTP_TIMEOUT', 'ca_certificates_file': 'CLOUDSDK_CORE_CUSTOM_CA_CERTS_FILE', 'max_retry_delay': 'CLOUDSDK_STORAGE_BASE_RETRY_DELAY', 'num_retries': 'CLOUDSDK_STORAGE_MAX_RETRIES', }, 'GSUtil': { 'check_hashes': 'CLOUDSDK_STORAGE_CHECK_HASHES', 'default_project_id': 'CLOUDSDK_CORE_PROJECT', 'disable_analytics_prompt': 'CLOUDSDK_CORE_DISABLE_USAGE_REPORTING', 'use_magicfile': 'CLOUDSDK_STORAGE_USE_MAGICFILE', 'parallel_composite_upload_threshold': 'CLOUDSDK_STORAGE_PARALLEL_COMPOSITE_UPLOAD_THRESHOLD', 'resumable_threshold': 'CLOUDSDK_STORAGE_RESUMABLE_THRESHOLD', 'rsync_buffer_lines': 'CLOUDSDK_STORAGE_RSYNC_LIST_CHUNK_SIZE', }, 'OAuth2': { 'client_id': 'CLOUDSDK_AUTH_CLIENT_ID', 'client_secret': 'CLOUDSDK_AUTH_CLIENT_SECRET', 'provider_authorization_uri': 'CLOUDSDK_AUTH_AUTH_HOST', 'provider_token_uri': 'CLOUDSDK_AUTH_TOKEN_HOST', }, } _REQUIRED_BOTO_CONFIG_NOT_YET_SUPPORTED = frozenset( # TOTO(b/214245419) Remove this once STET is support and add equivalent # mapping above. ['stet_binary_path', 'stet_config_path']) def get_flag_from_header(raw_header_key, header_value, unset=False): """Returns the gcloud storage flag for the given gsutil header. Args: raw_header_key: The header key. header_value: The header value unset: If True, the equivalent clear/remove flag is returned instead of the setter flag. This only applies to setmeta. Returns: A string representing the equivalent gcloud storage flag and value, if translation is possible, else returns None. Examples: >> get_flag_from_header('Cache-Control', 'val') --cache-control=val >> get_flag_from_header('x-goog-meta-foo', 'val') --update-custom-metadata=foo=val >> get_flag_from_header('x-goog-meta-foo', 'val', unset=True) --remove-custom-metadata=foo """ lowercase_header_key = raw_header_key.lower() if lowercase_header_key in PRECONDITIONS_HEADERS: providerless_flag = raw_header_key[len('x-goog-'):] if not providerless_flag.startswith('if-'): flag_name = 'if-' + providerless_flag else: flag_name = providerless_flag elif lowercase_header_key in DATA_TRANSFER_HEADERS: flag_name = lowercase_header_key else: flag_name = None if flag_name is not None: if unset: if lowercase_header_key in PRECONDITIONS_HEADERS or lowercase_header_key == 'content-md5': # Precondition headers and content-md5 cannot be cleared. return None else: return '--clear-' + flag_name return '--{}={}'.format(flag_name, header_value) for header_prefix in ('x-goog-meta-', 'x-amz-meta-'): if lowercase_header_key.startswith(header_prefix): # Preserving raw header keys to avoid forcing lowercase on custom data. metadata_key = raw_header_key[len(header_prefix):] if unset: return '--remove-custom-metadata=' + metadata_key else: return '--update-custom-metadata={}={}'.format(metadata_key, header_value) return None def get_format_flag_caret(): if system_util.IS_WINDOWS: return '^^^^' return '^' def get_format_flag_newline(): if system_util.IS_WINDOWS: return '^\\^n' return '\n' class GcloudStorageFlag(object): def __init__(self, gcloud_flag, repeat_type=None, supports_output_translation=False): """Initializes GcloudStorageFlag. Args: gcloud_flag (str\|dict): The name of the gcloud flag or a dictionary for when the gcloud flag depends on a gsutil value. gsutil "--pap off" -> gcloud "--no-public-access-prevention" repeat_type (RepeatFlagType\|None): Gsutil sometimes handles list and dictionary inputs by accepting a flag multiple times. support_output_translation (bool): If True, this flag in gcloud storage supports printing gsutil formatted output. """ self.gcloud_flag = gcloud_flag self.repeat_type = repeat_type self.supports_output_translation = supports_output_translation class GcloudStorageMap(object): """Mapping to translate gsutil command to its gcloud storage equivalent.""" def __init__(self, gcloud_command, flag_map, supports_output_translation=False): """Intalizes GcloudStorageMap. Args: gcloud_command (dict\|str): The corresponding name of the command to be called in gcloud. If this command supports sub-commands, then this field must be a dict of sub-command-name:GcloudStorageMap pairs. flag_map (dict): A dict of str to GcloudStorageFlag. Mapping of gsutil flags to their equivalent gcloud storage flag names. supports_output_translation (bool): Indicates if the corresponding gcloud storage command supports the printing gsutil formatted output. """ self.gcloud_command = gcloud_command self.flag_map = flag_map self.supports_output_translation = supports_output_translation def _get_gcloud_binary_path(cloudsdk_root): return os.path.join(cloudsdk_root, 'bin', 'gcloud.cmd' if system_util.IS_WINDOWS else 'gcloud') def _get_validated_gcloud_binary_path(): # GCLOUD_BINARY_PATH is used for testing purpose only. # It helps to run the parity_check.py script directly without having # to build gcloud. gcloud_binary_path = os.environ.get('GCLOUD_BINARY_PATH') if gcloud_binary_path: return gcloud_binary_path cloudsdk_root = os.environ.get('CLOUDSDK_ROOT_DIR') if cloudsdk_root is None: raise exception.GcloudStorageTranslationError( 'Requested to use "gcloud storage" but the gcloud binary path cannot' ' be found. This might happen if you attempt to use gsutil that was' ' not installed via Cloud SDK. You can manually set the' ' `CLOUDSDK_ROOT_DIR` environment variable to point to the' ' google-cloud-sdk installation directory to resolve the issue.' ' Alternatively, you can set `use_gcloud_storage=False` to disable' ' running the command using gcloud storage.') return _get_gcloud_binary_path(cloudsdk_root) def _get_gcs_json_endpoint_from_boto_config(config): gs_json_host = config.get('Credentials', 'gs_json_host') if gs_json_host: gs_json_port = config.get('Credentials', 'gs_json_port') port = ':' + gs_json_port if gs_json_port else '' json_api_version = config.get('Credentials', 'json_api_version', 'v1') return 'https://{}{}/storage/{}'.format(gs_json_host, port, json_api_version) return None def _get_s3_endpoint_from_boto_config(config): s3_host = config.get('Credentials', 's3_host') if s3_host: s3_port = config.get('Credentials', 's3_port') port = ':' + s3_port if s3_port else '' return 'https://{}{}'.format(s3_host, port) return None def _convert_args_to_gcloud_values(args, gcloud_storage_map): gcloud_args = [] repeat_flag_data = collections.defaultdict(list) i = 0 while i < len(args): if args[i] not in gcloud_storage_map.flag_map: # Add raw value (positional args and flag values for non-repeated flags). gcloud_args.append(args[i]) i += 1 continue gcloud_flag_object = gcloud_storage_map.flag_map[args[i]] if not gcloud_flag_object: # Flag asked to be skipped over. i += 1 elif gcloud_flag_object.repeat_type: # Capture "v1" and "v2" in ["-k", "v1", "-k", "v2"]. repeat_flag_data[gcloud_flag_object].append(args[i + 1]) i += 2 elif isinstance(gcloud_flag_object.gcloud_flag, str): # Simple translation. # gsutil: "-x" -> gcloud: "-y" gcloud_args.append(gcloud_flag_object.gcloud_flag) i += 1 else: # isinstance(gcloud_flag_object.gcloud_flag, dict) # gsutil: "--pap on" -> gcloud: "--pap" # gsutil: "--pap off" -> gcloud: "--no-pap" if args[i + 1] not in gcloud_flag_object.gcloud_flag: raise ValueError( 'Flag value not in translation map for "{}": {}'.format( args[i], args[i + 1])) translated_flag_and_value = gcloud_flag_object.gcloud_flag[args[i + 1]] if translated_flag_and_value: gcloud_args.append(translated_flag_and_value) i += 2 for gcloud_flag_object, values in repeat_flag_data.items(): if gcloud_flag_object.repeat_type is RepeatFlagType.LIST: # gsutil: "-k v1 -k v2" -> gcloud: "-k=v1,v2" condensed_flag_values = ','.join(values) elif gcloud_flag_object.repeat_type is RepeatFlagType.DICT: # gcloud: "-d k1:v1 -d k2:v2" -> gcloud: "-d=k1=v1,k2=v2" condensed_flag_values = ','.join( ['{}={}'.format(*s.split(':', 1)) for s in values]) else: raise ValueError('Shim cannot handle repeat flag type: {}'.format( repeat_flag_data.repeat_type)) gcloud_args.append('{}={}'.format(gcloud_flag_object.gcloud_flag, condensed_flag_values)) return gcloud_args class GcloudStorageCommandMixin(object): """Provides gcloud storage translation functionality. The command.Command class must inherit this class in order to support converting the gsutil command to it's gcloud storage equivalent. """ # Mapping for translating gsutil command to gcloud storage. gcloud_storage_map = None def __init__(self): self._translated_gcloud_storage_command = None self._translated_env_variables = None self._gcloud_has_active_account = None def gcloud_has_active_account(self): """Returns True if gcloud has an active account configured.""" gcloud_path = _get_validated_gcloud_binary_path() if self._gcloud_has_active_account is None: process = subprocess.run([gcloud_path, 'config', 'get', 'account'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') if process.returncode: raise exception.GcloudStorageTranslationError( "Error occurred while trying to retrieve gcloud's active account." " Error: {}".format(process.stderr)) # stdout will have the name of the account if active account is available. # Empty string indicates no active account available. self._gcloud_has_active_account = process.stdout.strip() != '' self.logger.debug('Result for "gcloud config get account" command:\n' ' STDOUT: {}.\n STDERR: {}'.format( process.stdout, process.stderr)) return self._gcloud_has_active_account def _get_gcloud_storage_args(self, sub_opts, gsutil_args, gcloud_storage_map): if gcloud_storage_map is None: raise exception.GcloudStorageTranslationError( 'Command "{}" cannot be translated to gcloud storage because the' ' translation mapping is missing.'.format(self.command_name)) args = [] if isinstance(gcloud_storage_map.gcloud_command, list): args.extend(gcloud_storage_map.gcloud_command) elif isinstance(gcloud_storage_map.gcloud_command, dict): # If a command has sub-commands, e.g gsutil pap set, gsutil pap get. # All the flags mapping must be present in the subcommand's map # because gsutil does not have command specific flags # if sub-commands are present. if gcloud_storage_map.flag_map: raise ValueError( 'Flags mapping should not be present at the top-level command if ' 'a sub-command is used. Command: {}.'.format(self.command_name)) sub_command = gsutil_args[0] sub_opts, parsed_args = self.ParseSubOpts( args=gsutil_args[1:], should_update_sub_opts_and_args=False) return self._get_gcloud_storage_args( sub_opts, parsed_args, gcloud_storage_map.gcloud_command.get(sub_command)) else: raise ValueError('Incorrect mapping found for "{}" command'.format( self.command_name)) if sub_opts: for option, value in sub_opts: if option not in gcloud_storage_map.flag_map: raise exception.GcloudStorageTranslationError( 'Command option "{}" cannot be translated to' ' gcloud storage'.format(option)) else: args.append(option) if value != '': # Empty string represents that the user did not passed in a value # for the flag. args.append(value) return _convert_args_to_gcloud_values(args + gsutil_args, gcloud_storage_map) def _translate_top_level_flags(self): """Translates gsutil's top level flags. Gsutil specifies the headers (-h) and boto config (-o) as top level flags as well, but we handle those separately. Returns: A tuple. The first item is a list of top level flags that can be appended to the gcloud storage command. The second item is a dict of environment variables that can be set for the gcloud storage command execution. """ top_level_flags = [] env_variables = { 'CLOUDSDK_METRICS_ENVIRONMENT': 'gsutil_shim', 'CLOUDSDK_STORAGE_RUN_BY_GSUTIL_SHIM': 'True' } if self.debug >= 3: top_level_flags.extend(['--verbosity', 'debug']) if self.debug == 4: top_level_flags.append('--log-http') if self.quiet_mode: top_level_flags.append('--no-user-output-enabled') if self.user_project: top_level_flags.append('--billing-project=' + self.user_project) if self.trace_token: top_level_flags.append('--trace-token=' + self.trace_token) if constants.IMPERSONATE_SERVICE_ACCOUNT: top_level_flags.append('--impersonate-service-account=' + constants.IMPERSONATE_SERVICE_ACCOUNT) # TODO(b/208294509) Add --perf-trace-token translation. should_use_rsync_override = self.command_name == 'rsync' and not ( config.get('GSUtil', 'parallel_process_count') == '1' and config.get('GSUtil', 'thread_process_count') == '1') if not (self.parallel_operations or should_use_rsync_override): # TODO(b/208301084) Set the --sequential flag instead. env_variables['CLOUDSDK_STORAGE_THREAD_COUNT'] = '1' env_variables['CLOUDSDK_STORAGE_PROCESS_COUNT'] = '1' return top_level_flags, env_variables def _translate_headers(self, headers=None, unset=False): """Translates gsutil headers to equivalent gcloud storage flags. Args: headers (dict\|None): If absent, extracts headers from command instance. unset (bool): Yield metadata clear flags instead of setter flags. Returns: List[str]: Translated flags for gcloud. Raises: GcloudStorageTranslationError: Could not translate flag. """ flags = [] # Accept custom headers or extract headers dict from Command class. headers_to_translate = headers if headers is not None else self.headers additional_headers = [] for raw_header_key, header_value in headers_to_translate.items(): lowercase_header_key = raw_header_key.lower() if lowercase_header_key == 'x-goog-api-version': # Gsutil adds this header. We don't have to translate it for gcloud. continue flag = get_flag_from_header(raw_header_key, header_value, unset=unset) if self.command_name in COMMANDS_SUPPORTING_ALL_HEADERS: if flag: flags.append(flag) elif (self.command_name in PRECONDITONS_ONLY_SUPPORTED_COMMANDS and lowercase_header_key in PRECONDITIONS_HEADERS): flags.append(flag) if not flag: self.logger.warn('Header {}:{} cannot be translated to a gcloud storage' ' equivalent flag. It is being treated as an arbitrary' ' request header.'.format(raw_header_key, header_value)) additional_headers.append('{}={}'.format(raw_header_key, header_value)) if additional_headers: flags.append('--additional-headers=' + ','.join(additional_headers)) return flags def _translate_boto_config(self): """Translates boto config options to gcloud storage properties. Returns: A tuple where first element is a list of flags and the second element is a dict representing the env variables that can be set to set the gcloud storage properties. """ flags = [] env_vars = {} # Handle gs_json_host and gs_json_port. gcs_json_endpoint = _get_gcs_json_endpoint_from_boto_config(config) if gcs_json_endpoint: env_vars['CLOUDSDK_API_ENDPOINT_OVERRIDES_STORAGE'] = gcs_json_endpoint # Handle s3_host and s3_port. s3_endpoint = _get_s3_endpoint_from_boto_config(config) if s3_endpoint: env_vars['CLOUDSDK_STORAGE_S3_ENDPOINT_URL'] = s3_endpoint decryption_keys = [] for section_name, section in config.items(): for key, value in section.items(): if (key == 'encryption_key' and self.command_name in ENCRYPTION_SUPPORTED_COMMANDS): flags.append('--encryption-key=' + value) # Boto config can have decryption keys in the form of # decryption_key1..100. elif (DECRYPTION_KEY_REGEX.match(key) and self.command_name in ENCRYPTION_SUPPORTED_COMMANDS): decryption_keys.append(value) elif (key == 'content_language' and self.command_name in COMMANDS_SUPPORTING_ALL_HEADERS): flags.append('--content-language=' + value) elif key in _REQUIRED_BOTO_CONFIG_NOT_YET_SUPPORTED: self.logger.error('The boto config field {}:{} cannot be translated' ' to gcloud storage equivalent.'.format( section_name, key)) elif key == 'https_validate_certificates' and not value: env_vars['CLOUDSDK_AUTH_DISABLE_SSL_VALIDATION'] = True # Skip mapping GS HMAC auth keys if gsutil wouldn't use them. elif (key in ('gs_access_key_id', 'gs_secret_access_key') and not boto_util.UsingGsHmac()): self.logger.debug('The boto config field {}:{} skipped translation' ' to the gcloud storage equivalent as it would' ' have been unused in gsutil.'.format( section_name, key)) else: env_var = _BOTO_CONFIG_MAP.get(section_name, {}).get(key, None) if env_var is not None: env_vars[env_var] = value if decryption_keys: flags.append('--decryption-keys=' + ','.join(decryption_keys)) return flags, env_vars def get_gcloud_storage_args(self, gcloud_storage_map=None): """Translates the gsutil command flags to gcloud storage flags. It uses the command_spec.gcloud_storage_map field that provides the translation mapping for all the flags. Args: gcloud_storage_map (GcloudStorageMap\|None): Command surface may pass a custom translation map instead of using the default class constant. Useful for when translations change based on conditional logic. Returns: A list of all the options and arguments that can be used with the equivalent gcloud storage command. Raises: GcloudStorageTranslationError: If a flag or command cannot be translated. ValueError: If there is any issue with the mapping provided by GcloudStorageMap. """ return self._get_gcloud_storage_args( self.sub_opts, self.args, gcloud_storage_map or self.gcloud_storage_map) def _print_gcloud_storage_command_info(self, gcloud_command, env_variables, dry_run=False): logger_func = self.logger.info if dry_run else self.logger.debug logger_func('Gcloud Storage Command: {}'.format(' '.join(gcloud_command))) if env_variables: logger_func('Environment variables for Gcloud Storage:') for k, v in env_variables.items(): logger_func('%s=%s', k, v) def _get_full_gcloud_storage_execution_information(self, args): top_level_flags, env_variables = self._translate_top_level_flags() header_flags = self._translate_headers() flags_from_boto, env_vars_from_boto = self._translate_boto_config() env_variables.update(env_vars_from_boto) gcloud_binary_path = _get_validated_gcloud_binary_path() gcloud_storage_command = ([gcloud_binary_path] + args + top_level_flags + header_flags + flags_from_boto) return env_variables, gcloud_storage_command def translate_to_gcloud_storage_if_requested(self): """Translates the gsutil command to gcloud storage equivalent. The translated commands get stored at self._translated_gcloud_storage_command. This command also translate the boto config, which gets stored as a dict at self._translated_env_variables Returns: True if the command was successfully translated, else False. """ if self.command_name == 'version' or self.command_name == 'test': # Running any command in debug mode will lead to calling gsutil version # command. We don't want to translate the version command as this # should always reflect the version that gsutil is using. # We don't want to run any translation for the "test" command. return False use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage', False) try: hidden_shim_mode = HIDDEN_SHIM_MODE( config.get('GSUtil', 'hidden_shim_mode', 'none')) except ValueError: raise exception.CommandException( 'Invalid option specified for' ' GSUtil:hidden_shim_mode config setting. Should be one of: {}'. format(' \| '.join([x.value for x in HIDDEN_SHIM_MODE]))) if use_gcloud_storage: try: env_variables, gcloud_storage_command = self._get_full_gcloud_storage_execution_information( self.get_gcloud_storage_args()) if not self.gcloud_has_active_account(): # Allow running gcloud with anonymous credentials. env_variables['CLOUDSDK_AUTH_DISABLE_CREDENTIALS'] = 'True' if hidden_shim_mode == HIDDEN_SHIM_MODE.DRY_RUN: self._print_gcloud_storage_command_info(gcloud_storage_command, env_variables, dry_run=True) elif not os.environ.get('CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL'): raise exception.GcloudStorageTranslationError( 'Requested to use "gcloud storage" but gsutil is not using the' ' same credentials as gcloud.' ' You can make gsutil use the same credentials by running:\n' '{} config set pass_credentials_to_gsutil True'.format( _get_validated_gcloud_binary_path())) elif (boto_util.UsingGsHmac() and ApiSelector.XML not in self.command_spec.gs_api_support): raise CommandException( 'Requested to use "gcloud storage" with Cloud Storage XML API' ' HMAC credentials but the "{}" command can only be used' ' with the Cloud Storage JSON API.'.format(self.command_name)) else: self._print_gcloud_storage_command_info(gcloud_storage_command, env_variables) self._translated_gcloud_storage_command = gcloud_storage_command self._translated_env_variables = env_variables return True except exception.GcloudStorageTranslationError as e: # Raise error if no_fallback mode has been requested. This mode # should only be used for debuggling and testing purposes. if hidden_shim_mode == HIDDEN_SHIM_MODE.NO_FALLBACK: raise exception.CommandException(e) # For all other cases, we want to run gsutil. self.logger.error( 'Cannot translate gsutil command to gcloud storage.' ' Going to run gsutil command. Error: %s', e) return False def _get_shim_command_environment_variables(self): subprocess_envs = os.environ.copy() subprocess_envs.update(self._translated_env_variables) return subprocess_envs def run_gcloud_storage(self): process = subprocess.run(self._translated_gcloud_storage_command, env=self._get_shim_command_environment_variables()) return process.returncode	27.25 KB	2025-06-25 08:19
📄 signurl_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions for signurl command.""" import base64 from datetime import datetime from datetime import timezone import hashlib from gslib.utils.constants import UTF8 import six from six.moves import urllib _CANONICAL_REQUEST_FORMAT = ('{method}\n{resource}\n{query_string}\n{headers}' '\n{signed_headers}\n{hashed_payload}') _SIGNING_ALGO = 'GOOG4-RSA-SHA256' _STRING_TO_SIGN_FORMAT = ('{signing_algo}\n{request_time}\n{credential_scope}' '\n{hashed_request}') _SIGNED_URL_FORMAT = ('https://{host}/{path}?x-goog-signature={sig}&' '{query_string}') _UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD' def _NowUTC(): return datetime.now(tz=timezone.utc).replace(tzinfo=None) def CreatePayload(client_id, method, duration, path, generation, logger, region, signed_headers, billing_project=None, string_to_sign_debug=False): """Create a string that needs to be signed. Args: client_id: Client ID signing this URL. method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket of object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. billing_project: Specify a user project to be billed for the request. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Returns: A tuple where the 1st element is the string to sign. The second element is the query string. """ signing_time = _NowUTC() canonical_day = signing_time.strftime('%Y%m%d') canonical_time = signing_time.strftime('%Y%m%dT%H%M%SZ') canonical_scope = '{date}/{region}/storage/goog4_request'.format( date=canonical_day, region=region) signed_query_params = { 'x-goog-algorithm': _SIGNING_ALGO, 'x-goog-credential': client_id + '/' + canonical_scope, 'x-goog-date': canonical_time, 'x-goog-signedheaders': ';'.join(sorted(signed_headers.keys())), 'x-goog-expires': '%d' % duration.total_seconds() } if (billing_project is not None): signed_query_params['userProject'] = billing_project if generation is not None: signed_query_params['generation'] = generation canonical_resource = '/{}'.format(path) canonical_query_string = '&'.join([ '{}={}'.format(param, urllib.parse.quote_plus(signed_query_params[param])) for param in sorted(signed_query_params.keys()) ]) canonical_headers = '\n'.join([ '{}:{}'.format(header.lower(), signed_headers[header]) for header in sorted(signed_headers.keys()) ]) + '\n' canonical_signed_headers = ';'.join(sorted(signed_headers.keys())) canonical_request = _CANONICAL_REQUEST_FORMAT.format( method=method, resource=canonical_resource, query_string=canonical_query_string, headers=canonical_headers, signed_headers=canonical_signed_headers, hashed_payload=_UNSIGNED_PAYLOAD) if six.PY3: canonical_request = canonical_request.encode(UTF8) canonical_request_hasher = hashlib.sha256() canonical_request_hasher.update(canonical_request) hashed_canonical_request = base64.b16encode( canonical_request_hasher.digest()).lower().decode(UTF8) string_to_sign = _STRING_TO_SIGN_FORMAT.format( signing_algo=_SIGNING_ALGO, request_time=canonical_time, credential_scope=canonical_scope, hashed_request=hashed_canonical_request) if string_to_sign_debug and logger: logger.debug( 'Canonical request (ignore opening/closing brackets): [[[%s]]]' % canonical_request) logger.debug('String to sign (ignore opening/closing brackets): [[[%s]]]' % string_to_sign) return string_to_sign, canonical_query_string def GetFinalUrl(raw_signature, host, path, canonical_query_string): """Get the final signed url.""" signature = base64.b16encode(raw_signature).lower().decode() return _SIGNED_URL_FORMAT.format(host=host, path=path, sig=signature, query_string=canonical_query_string) def to_bytes(value, encoding=UTF8): if isinstance(value, bytes): return value return value.encode(encoding)	5.34 KB	2025-06-25 08:19
📄 stet_util.py # -- coding: utf-8 -- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for Split Trust Encryption Tool integration.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import shutil from gslib import storage_url from gslib.utils import execution_util from gslib.utils import temporary_file_util from boto import config class StetSubcommandName(object): """Enum class for available STET subcommands.""" ENCRYPT = 'encrypt' DECRYPT = 'decrypt' def _get_stet_binary_from_path(): """Retrieves STET binary from path if available. Python 2 compatible.""" for path_directory in os.getenv('PATH').split(os.path.pathsep): binary_path = os.path.join(path_directory, 'stet') if os.path.exists(binary_path): return binary_path def _stet_transform(subcommand, blob_id, in_file_path, out_file_path, logger): """Runs a STET transform on a file. Encrypts for uploads. Decrypts for downloads. Automatically populates flags for the STET binary. Args: subcommand (StetSubcommandName): Subcommand to call on STET binary. blob_id (str): Cloud URL that binary uses for validation. in_file_path (str): File to be transformed source. out_file_path (str): Where to write result of transform. logger (logging.Logger): For logging STET binary output. Raises: KeyError: STET binary or config could not be found. """ binary_path = config.get('GSUtil', 'stet_binary_path', _get_stet_binary_from_path()) if not binary_path: raise KeyError('Could not find STET binary in boto config or PATH.') command_args = [os.path.expanduser(binary_path), subcommand] config_path = config.get('GSUtil', 'stet_config_path', None) if config_path: command_args.append('--config-file=' + os.path.expanduser(config_path)) command_args.extend(['--blob-id=' + blob_id, in_file_path, out_file_path]) _, stderr = execution_util.ExecuteExternalCommand(command_args) logger.debug(stderr) def encrypt_upload(source_url, destination_url, logger): """Encrypts a file with STET binary before upload. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. logger (logging.Logger): For logging STET binary output. Returns: stet_temporary_file_url (StorageUrl): Path to STET-encrypted file. """ in_file = source_url.object_name out_file = temporary_file_util.GetStetTempFileName(source_url) blob_id = destination_url.url_string _stet_transform(StetSubcommandName.ENCRYPT, blob_id, in_file, out_file, logger) return storage_url.StorageUrlFromString(out_file) def decrypt_download(source_url, destination_url, temporary_file_name, logger): """STET-decrypts downloaded file. Args: source_url (StorageUrl): Copy source. destination_url (StorageUrl): Copy destination. temporary_file_name (str): Path to temporary file used for download. logger (logging.Logger): For logging STET binary output. """ in_file = temporary_file_name out_file = temporary_file_util.GetStetTempFileName(destination_url) blob_id = source_url.url_string _stet_transform(StetSubcommandName.DECRYPT, blob_id, in_file, out_file, logger) shutil.move(out_file, in_file)	3.83 KB	2025-06-25 08:19
📄 system_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for interacting with the host system. The methods in this module should be limited to obtaining system information and simple file operations (disk info, retrieving metadata about existing files, creating directories, fetching environment variables, etc.). """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import locale import os import struct import sys import six from gslib.utils.constants import WINDOWS_1252 _DEFAULT_NUM_TERM_LINES = 25 PLATFORM = str(sys.platform).lower() # Detect platform types. IS_WINDOWS = 'win32' in PLATFORM IS_CYGWIN = 'cygwin' in PLATFORM IS_LINUX = 'linux' in PLATFORM IS_OSX = 'darwin' in PLATFORM # pylint: disable=g-import-not-at-top if IS_WINDOWS: from ctypes import c_int from ctypes import c_uint64 from ctypes import c_char_p from ctypes import c_wchar_p from ctypes import windll from ctypes import POINTER from ctypes import WINFUNCTYPE from ctypes import WinError IS_CP1252 = locale.getdefaultlocale()[1] == WINDOWS_1252 else: IS_CP1252 = False def CheckFreeSpace(path): """Return path/drive free space (in bytes).""" if IS_WINDOWS: try: # pylint: disable=invalid-name get_disk_free_space_ex = WINFUNCTYPE(c_int, c_wchar_p, POINTER(c_uint64), POINTER(c_uint64), POINTER(c_uint64)) get_disk_free_space_ex = get_disk_free_space_ex( ('GetDiskFreeSpaceExW', windll.kernel32), ( (1, 'lpszPathName'), (2, 'lpFreeUserSpace'), (2, 'lpTotalSpace'), (2, 'lpFreeSpace'), )) except AttributeError: get_disk_free_space_ex = WINFUNCTYPE(c_int, c_char_p, POINTER(c_uint64), POINTER(c_uint64), POINTER(c_uint64)) get_disk_free_space_ex = get_disk_free_space_ex( ('GetDiskFreeSpaceExA', windll.kernel32), ( (1, 'lpszPathName'), (2, 'lpFreeUserSpace'), (2, 'lpTotalSpace'), (2, 'lpFreeSpace'), )) def GetDiskFreeSpaceExErrCheck(result, unused_func, args): if not result: raise WinError() return args[1].value get_disk_free_space_ex.errcheck = GetDiskFreeSpaceExErrCheck return get_disk_free_space_ex(os.getenv('SystemDrive')) else: (_, f_frsize, _, _, f_bavail, _, _, _, _, _) = os.statvfs(path) return f_frsize * f_bavail def CloudSdkCredPassingEnabled(): return os.environ.get('CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL') == '1' def CloudSdkVersion(): return os.environ.get('CLOUDSDK_VERSION', '') def CreateDirIfNeeded(dir_path, mode=0o777): """Creates a directory, suppressing already-exists errors.""" if not os.path.exists(dir_path): try: # Unfortunately, even though we catch and ignore EEXIST, this call will # output a (needless) error message (no way to avoid that in Python). os.makedirs(dir_path, mode) # Ignore 'already exists' in case user tried to start up several # resumable uploads concurrently from a machine where no tracker dir had # yet been created. except OSError as e: if e.errno != errno.EEXIST and e.errno != errno.EISDIR: raise def GetDiskCounters(): """Retrieves disk I/O statistics for all disks. Adapted from the psutil module's psutil._pslinux.disk_io_counters: http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py Originally distributed under under a BSD license. Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. Returns: A dictionary containing disk names mapped to the disk counters from /disk/diskstats. """ # iostat documentation states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. This value is needed to # calculate the amount of disk I/O in bytes. sector_size = 512 partitions = [] with open('/proc/partitions', 'r') as f: lines = f.readlines()[2:] for line in lines: _, _, _, name = line.split() if name[-1].isdigit(): partitions.append(name) retdict = {} with open('/proc/diskstats', 'r') as f: for line in f: values = line.split()[:11] _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values if name in partitions: rbytes = int(rbytes) * sector_size wbytes = int(wbytes) * sector_size reads = int(reads) writes = int(writes) rtime = int(rtime) wtime = int(wtime) retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime) return retdict def GetFileSize(fp, position_to_eof=False): """Returns size of file, optionally leaving fp positioned at EOF.""" if not position_to_eof: cur_pos = fp.tell() fp.seek(0, os.SEEK_END) cur_file_size = fp.tell() if not position_to_eof: fp.seek(cur_pos) return cur_file_size def GetGsutilClientIdAndSecret(): """Returns a tuple of the gsutil OAuth2 client ID and secret. Google OAuth2 clients always have a secret, even if the client is an installed application/utility such as gsutil. Of course, in such cases the "secret" is actually publicly known; security depends entirely on the secrecy of refresh tokens, which effectively become bearer tokens. Returns: (str, str) A 2-tuple of (client ID, secret). """ if InvokedViaCloudSdk() and CloudSdkCredPassingEnabled(): # Cloud SDK installs have a separate client ID / secret. return ( '32555940559.apps.googleusercontent.com', # Cloud SDK client ID 'ZmssLNjJy2998hD4CTg2ejr2') # Cloud SDK secret return ( '909320924072.apps.googleusercontent.com', # gsutil client ID 'p3RlpR10xMFh9ZXBS/ZNLYUu') # gsutil secret def GetStreamFromFileUrl(storage_url, mode='rb'): if storage_url.IsStream(): return sys.stdin if six.PY2 else sys.stdin.buffer else: return open(storage_url.object_name, mode) def GetTermLines(): """Returns number of terminal lines.""" # fcntl isn't supported in Windows. try: import fcntl # pylint: disable=g-import-not-at-top import termios # pylint: disable=g-import-not-at-top except ImportError: return _DEFAULT_NUM_TERM_LINES def ioctl_GWINSZ(fd): # pylint: disable=invalid-name try: return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))[0] except: # pylint: disable=bare-except return 0 # Failure (so will retry on different file descriptor below). # Try to find a valid number of lines from termio for stdin, stdout, # or stderr, in that order. ioc = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not ioc: try: fd = os.open(os.ctermid(), os.O_RDONLY) ioc = ioctl_GWINSZ(fd) os.close(fd) except: # pylint: disable=bare-except pass if not ioc: ioc = os.environ.get('LINES', _DEFAULT_NUM_TERM_LINES) return int(ioc) def InvokedViaCloudSdk(): return os.environ.get('CLOUDSDK_WRAPPER') == '1' def IsRunningInCiEnvironment(): """Returns True if running in a CI environment, e.g. GitHub CI.""" # https://docs.github.com/en/actions/reference/environment-variables on_github_ci = 'CI' in os.environ on_kokoro = 'KOKORO_ROOT' in os.environ return on_github_ci or on_kokoro def IsRunningInteractively(): """Returns True if currently running interactively on a TTY.""" return sys.stdout.isatty() and sys.stderr.isatty() and sys.stdin.isatty() def MonkeyPatchHttp(): ver = sys.version_info # Checking for and applying monkeypatch for Python versions: # 3.0 - 3.6.6, 3.7.0 if ver.major == 3: if (ver.minor < 6 or (ver.minor == 6 and ver.micro < 7) or (ver.minor == 7 and ver.micro == 0)): _MonkeyPatchHttpForPython_3x() def _MonkeyPatchHttpForPython_3x(): # We generally have to do all sorts of gross things when applying runtime # patches (dynamic imports, invalid names to resolve symbols in copy/pasted # methods, invalid spacing from copy/pasted methods, etc.), so we just disable # pylint warnings for this whole method. # pylint: disable=all # This fixes https://bugs.python.org/issue33365. A fix was applied in # https://github.com/python/cpython/commit/936f03e7fafc28fd6fdfba11d162c776b89c0167 # but to apply that at runtime would mean patching the entire begin() method. # Rather, we just override begin() to call its old self, followed by printing # the HTTP headers afterward. This prevents us from overriding more behavior # than we have to. import http old_begin = http.client.HTTPResponse.begin def PatchedBegin(self): old_begin(self) if self.debuglevel > 0: for hdr, val in self.headers.items(): print("header:", hdr + ":", val) http.client.HTTPResponse.begin = PatchedBegin def StdinIterator(): """A generator function that returns lines from stdin.""" for line in sys.stdin: # Strip CRLF. yield line.rstrip() class StdinIteratorCls(six.Iterator): """An iterator that returns lines from stdin. This is needed because Python 3 balks at pickling the generator version above. """ def __iter__(self): return self def __next__(self): line = sys.stdin.readline() if not line: raise StopIteration() return line.rstrip()	9.8 KB	2025-06-25 08:19
📄 temporary_file_util.py # -- coding: utf-8 -- # Copyright 2021 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utils for temporary files.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals def GetTempFileName(storage_url): """Returns temporary file name for uncompressed file.""" return '%s_.gstmp' % storage_url.object_name def GetTempZipFileName(storage_url): """Returns temporary name for a temporarily compressed file.""" return '%s_.gztmp' % storage_url.object_name def GetStetTempFileName(storage_url): """Returns temporary file name for result of STET transform.""" return '%s_.stet_tmp' % storage_url.object_name	1.22 KB	2025-06-25 08:19
📄 text_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility structures and methods for manipulating text.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import binascii import codecs import os import sys import io import re import locale import collections import random import six import string from six.moves import urllib from six.moves import range from gslib.exception import CommandException from gslib.lazy_wrapper import LazyWrapper from gslib.utils.constants import UTF8 from gslib.utils.constants import WINDOWS_1252 from gslib.utils.system_util import IS_CP1252 if six.PY3: long = int STORAGE_CLASS_SHORTHAND_TO_FULL_NAME = { # Values should remain uppercase, as required by non-gs providers. 'CL': 'COLDLINE', 'DRA': 'DURABLE_REDUCED_AVAILABILITY', 'NL': 'NEARLINE', 'S': 'STANDARD', 'STD': 'STANDARD', 'A': 'ARCHIVE', } VERSION_MATCHER = LazyWrapper( lambda: re.compile(r'^(?P<maj>\d+)(\.(?P<min>\d+)(?P<suffix>.))?')) def AddQueryParamToUrl(url_str, param_name, param_value): """Adds a query parameter to a URL string. Appends a query parameter to the query string portion of a url. If a parameter with the given name was already present, it is not removed; the new name/value pair will be appended to the end of the query string. It is assumed that all arguments will be of type `str` (either ASCII or UTF-8 encoded) or `unicode`. Note that this method performs no URL-encoding. It is the caller's responsibility to ensure proper URL encoding of the entire URL; i.e. if the URL is already URL-encoded, you should pass in URL-encoded values for param_name and param_value. If the URL is not URL-encoded, you should not pass in URL-encoded parameters; instead, you could perform URL-encoding using the URL string returned from this function. Args: url_str: (str or unicode) String representing the URL. param_name: (str or unicode) String key of the query parameter. param_value: (str or unicode) String value of the query parameter. Returns: (str or unicode) A string representing the modified url, of type `unicode` if the url_str argument was a `unicode`, otherwise a `str` encoded in UTF-8. """ scheme, netloc, path, query_str, fragment = urllib.parse.urlsplit(url_str) query_params = urllib.parse.parse_qsl(query_str, keep_blank_values=True) query_params.append((param_name, param_value)) new_query_str = '&'.join(['%s=%s' % (k, v) for (k, v) in query_params]) new_url = urllib.parse.urlunsplit( (scheme, netloc, path, new_query_str, fragment)) return new_url def CompareVersions(first, second): """Compares the first and second gsutil version strings. For example, 3.33 > 3.7, and 4.1 is a greater major version than 3.33. Does not handle multiple periods (e.g. 3.3.4) or complicated suffixes (e.g., 3.3RC4 vs. 3.3RC5). A version string with a suffix is treated as less than its non-suffix counterpart (e.g. 3.32 > 3.32pre). Args: first: First gsutil version string. second: Second gsutil version string. Returns: (g, m): g is True if first known to be greater than second, else False. m is True if first known to be greater by at least 1 major version, else False. """ m1 = VERSION_MATCHER().match(str(first)) m2 = VERSION_MATCHER().match(str(second)) # If passed strings we don't know how to handle, be conservative. if not m1 or not m2: return (False, False) major_ver1 = int(m1.group('maj')) minor_ver1 = int(m1.group('min')) if m1.group('min') else 0 suffix_ver1 = m1.group('suffix') major_ver2 = int(m2.group('maj')) minor_ver2 = int(m2.group('min')) if m2.group('min') else 0 suffix_ver2 = m2.group('suffix') if major_ver1 > major_ver2: return (True, True) elif major_ver1 == major_ver2: if minor_ver1 > minor_ver2: return (True, False) elif minor_ver1 == minor_ver2: return (bool(suffix_ver2) and not suffix_ver1, False) return (False, False) def ConvertRecursiveToFlatWildcard(url_strs): """A generator that adds '' to each url string in url_strs.""" for url_str in url_strs: yield '%s' % url_str def DecodeLongAsString(long_to_convert): """Decodes an encoded python long into an ASCII string. This is used for modeling S3 version_id's as apitools generation. Args: long_to_convert: long to convert to ASCII string. If this is already a string, it is simply returned. Returns: String decoded from the input long. """ unhexed = binascii.unhexlify(hex(long_to_convert)[2:].rstrip('L')) return six.ensure_str(unhexed) def EncodeStringAsLong(string_to_convert): """Encodes an ASCII string as a python long. This is used for modeling S3 version_id's as apitools generation. Because python longs can be arbitrarily large, this works. Args: string_to_convert: ASCII string to convert to a long. Returns: Long that represents the input string. """ hex_bytestr = codecs.encode(six.ensure_binary(string_to_convert), 'hex_codec') # Note that `long`/`int` accepts either `bytes` or `unicode` as the # first arg in both py2 and py3: return long(hex_bytestr, 16) def FixWindowsEncodingIfNeeded(input_str): """Attempts to detect Windows CP1252 encoding and convert to UTF8. Windows doesn't provide a way to set UTF-8 for string encodings; you can set the system locale (see http://windows.microsoft.com/en-us/windows/change-system-locale#1TC=windows-7) but that takes you to a "Change system locale" dropdown that just lists languages (e.g., "English (United States)". Instead, we're forced to check if a encoding as UTF8 raises an exception and if so, try converting from CP1252 to Unicode. Args: input_str: (str or bytes) The input string. Returns: (unicode) The converted string or the original, if conversion wasn't needed. """ if IS_CP1252: return six.ensure_text(input_str, WINDOWS_1252) else: return six.ensure_text(input_str, UTF8) def GetPrintableExceptionString(exc): """Returns a short Unicode string describing the exception.""" return six.text_type(exc).encode(UTF8) or six.text_type(exc.__class__) def InsistAscii(string, message): """Ensures that the string passed in consists of only ASCII values. Args: string: Union[str, unicode, bytes] Text that will be checked for ASCII values. message: Union[str, unicode, bytes] Error message, passed into the exception, in the event that the check on `string` fails. Returns: None Raises: CommandException """ if not all(ord(c) < 128 for c in string): raise CommandException(message) def InsistAsciiHeader(header): """Checks for ASCII-only characters in `header`. Also constructs an error message using `header` if the check fails. Args: header: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None """ InsistAscii(header, 'Invalid non-ASCII header (%s).' % header) def InsistAsciiHeaderValue(header, value): """Checks for ASCII-only characters in `value`. Also constructs an error message using `header` and `value` if the check fails. Args: header: Header name, only used in error message in case of an exception. value: Union[str, binary, unicode] Text being checked for ASCII values. Returns: None """ InsistAscii( value, 'Invalid non-ASCII value (%s) was provided for header %s.\nOnly ASCII ' 'characters are allowed in headers other than x-goog-meta- and ' 'x-amz-meta- headers' % (repr(value), header)) def InsistOnOrOff(value, message): """Ensures that the value passed in consists of only "on" or "off" Args: value: (unicode) Unicode string that will be checked for correct text. message: Union[str, unicode, bytes] Error message passed into the exception in the event that the check on value fails. Returns: None Raises: CommandException """ if value != 'on' and value != 'off': raise CommandException(message) def NormalizeStorageClass(sc): """Returns a normalized form of the given storage class name. Converts the given string to uppercase and expands valid abbreviations to full storage class names (e.g 'std' would return 'STANDARD'). Note that this method does not check if the given storage class is valid. Args: sc: (str) String representing the storage class's full name or abbreviation. Returns: (str) A string representing the full name of the given storage class. """ # Use uppercase; storage class argument for the S3 API must be uppercase, # and it's case-insensitive for GS APIs. sc = sc.upper() if sc in STORAGE_CLASS_SHORTHAND_TO_FULL_NAME: sc = STORAGE_CLASS_SHORTHAND_TO_FULL_NAME[sc] return sc def PrintableStr(input_val): """Return an UTF8-encoded string type, or None if `input_val` is None. Args: input_val: (unicode, str, or None) A string-like object or None. This method simply calls encode() on `input_val` if it is not None; if `input_val` is not of type "unicode", this will implicitly call decode() with the default encoding for strings (for Python 2, this is ASCII), then call encode(). Returns: (str) A UTF-8 encoded string, or None. """ return input_val def print_to_fd(objects, kwargs): """A Python 2/3 compatible analogue to the print function. This function writes text to a file descriptor as the builtin print function would, favoring unicode encoding. Aguments and return values are the same as documented in the Python 2 print function. """ def _get_args(kwargs): """Validates keyword arguments that would be used in Print Valid keyword arguments, mirroring print(), are 'sep', 'end', and 'file'. These must be of types string, string, and file / file interface respectively. Returns the above kwargs of the above types. """ expected_keywords = collections.OrderedDict([('sep', ' '), ('end', '\n'), ('file', sys.stdout)]) for key, value in kwargs.items(): if key not in expected_keywords: error_msg = ('{} is not a valid keyword argument. ' 'Please use one of: {}') raise KeyError(error_msg.format(key, ' '.join(expected_keywords.keys()))) else: expected_keywords[key] = value return expected_keywords.values() def _get_byte_strings(objects): """Gets a `bytes` string for each item in a list of printable objects.""" byte_objects = [] for item in objects: if not isinstance(item, (six.binary_type, six.text_type)): # If the item wasn't bytes or unicode, its __str__ method # should return one of those types. item = str(item) if isinstance(item, six.binary_type): byte_objects.append(item) else: # The item should be unicode. If it's not, ensure_binary() # will throw a TypeError. byte_objects.append(six.ensure_binary(item)) return byte_objects sep, end, file = _get_args(kwargs) sep = six.ensure_binary(sep) end = six.ensure_binary(end) data = _get_byte_strings(objects) data = sep.join(data) data += end write_to_fd(file, data) def write_to_fd(fd, data): """Write given data to given file descriptor, doing any conversions needed""" if six.PY2: fd.write(data) return # PY3 logic: if isinstance(data, bytes): if (hasattr(fd, 'mode') and 'b' in fd.mode) or isinstance(fd, io.BytesIO): fd.write(data) elif hasattr(fd, 'buffer'): fd.buffer.write(data) else: fd.write(six.ensure_text(data)) elif 'b' in fd.mode: fd.write(six.ensure_binary(data)) else: fd.write(data) def RemoveCRLFFromString(input_str): r"""Returns the input string with all \n and \r removed.""" return re.sub(r'[\r\n]', '', input_str) def get_random_ascii_chars(size, seed=0): """Generates binary string representation of a list of ASCII characters. Args: size: Integer quantity of characters to generate. seed: A seed may be specified for deterministic behavior. Int 0 is used as the default value. Returns: Binary encoded string representation of a list of characters of length equal to size argument. """ random.seed(seed) contents = str([random.choice(string.ascii_letters) for _ in range(size)]) contents = six.ensure_binary(contents) random.seed() # Reset the seed for any other tests. return contents	12.99 KB	2025-06-25 08:19
📄 translation_helper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility module for translating XML API objects to/from JSON objects.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import json import re import textwrap import xml.etree.ElementTree from xml.etree.ElementTree import ParseError as XmlParseError import six from apitools.base.protorpclite.util import decode_datetime from apitools.base.py import encoding import boto from boto.gs.acl import ACL from boto.gs.acl import ALL_AUTHENTICATED_USERS from boto.gs.acl import ALL_USERS from boto.gs.acl import Entries from boto.gs.acl import Entry from boto.gs.acl import GROUP_BY_DOMAIN from boto.gs.acl import GROUP_BY_EMAIL from boto.gs.acl import GROUP_BY_ID from boto.gs.acl import USER_BY_EMAIL from boto.gs.acl import USER_BY_ID from boto.s3.tagging import Tags from boto.s3.tagging import TagSet from gslib.cloud_api import ArgumentException from gslib.cloud_api import BucketNotFoundException from gslib.cloud_api import NotFoundException from gslib.cloud_api import Preconditions from gslib.exception import CommandException from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import S3_ACL_MARKER_GUID from gslib.utils.constants import S3_MARKER_GUIDS if six.PY3: long = int CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I) CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I) CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I) CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I) CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I) CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I) CUSTOM_TIME_REGEX = re.compile(r'^custom-time', re.I) GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I) GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I) GOOG_METAGENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-metageneration-match', re.I) CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.)', re.I) CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.)', re.I) CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I) metadata_header_regexes = frozenset({ CACHE_CONTROL_REGEX, CONTENT_DISPOSITION_REGEX, CONTENT_ENCODING_REGEX, CONTENT_LANGUAGE_REGEX, CONTENT_MD5_REGEX, CONTENT_TYPE_REGEX, CUSTOM_TIME_REGEX, GOOG_API_VERSION_REGEX, GOOG_GENERATION_MATCH_REGEX, GOOG_METAGENERATION_MATCH_REGEX, CUSTOM_GOOG_METADATA_REGEX, CUSTOM_AMZ_METADATA_REGEX, }) # This distinguishes S3 custom headers from S3 metadata on objects. S3_HEADER_PREFIX = 'custom-amz-header' DEFAULT_CONTENT_TYPE = 'application/octet-stream' # Because CORS is just a list in apitools, we need special handling or blank # CORS lists will get sent with other configuration commands such as lifecycle, # which would cause CORS configuration to be unintentionally removed. # Protorpc defaults list values to an empty list, and won't allow us to set the # value to None like other configuration fields, so there is no way to # distinguish the default value from when we actually want to remove the CORS # configuration. To work around this, we create a dummy CORS entry that # signifies that we should nullify the CORS configuration. # A value of [] means don't modify the CORS configuration. # A value of REMOVE_CORS_CONFIG means remove the CORS configuration. REMOVE_CORS_CONFIG = [ apitools_messages.Bucket.CorsValueListEntry(maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG']) ] # Similar to CORS above, we need a sentinel value allowing us to specify # when a default object ACL should be private (containing no entries). # A defaultObjectAcl value of [] means don't modify the default object ACL. # A value of [PRIVATE_DEFAULT_OBJ_ACL] means create an empty/private default # object ACL. PRIVATE_DEFAULT_OBJ_ACL = apitools_messages.ObjectAccessControl( id='PRIVATE_DEFAULT_OBJ_ACL') def GetNonMetadataHeaders(headers): arbitrary_headers = {} for header, value in headers.items(): if not any(regex.match(header) for regex in metadata_header_regexes): arbitrary_headers[header] = value return arbitrary_headers def ObjectMetadataFromHeaders(headers): """Creates object metadata according to the provided headers. gsutil -h allows specifiying various headers (originally intended to be passed to boto in gsutil v3). For the JSON API to be compatible with this option, we need to parse these headers into gsutil_api Object fields. Args: headers: Dict of headers passed via gsutil -h Raises: ArgumentException if an invalid header is encountered. Returns: apitools Object with relevant fields populated from headers. """ obj_metadata = apitools_messages.Object() for header, value in headers.items(): if CACHE_CONTROL_REGEX.match(header): obj_metadata.cacheControl = value.strip() elif CONTENT_DISPOSITION_REGEX.match(header): obj_metadata.contentDisposition = value.strip() elif CONTENT_ENCODING_REGEX.match(header): obj_metadata.contentEncoding = value.strip() elif CONTENT_MD5_REGEX.match(header): obj_metadata.md5Hash = value.strip() elif CONTENT_LANGUAGE_REGEX.match(header): obj_metadata.contentLanguage = value.strip() elif CONTENT_TYPE_REGEX.match(header): if not value: obj_metadata.contentType = DEFAULT_CONTENT_TYPE else: obj_metadata.contentType = value.strip() elif CUSTOM_TIME_REGEX.match(header): obj_metadata.customTime = decode_datetime(value.strip()) elif GOOG_API_VERSION_REGEX.match(header): # API version is only relevant for XML, ignore and rely on the XML API # to add the appropriate version. continue elif GOOG_GENERATION_MATCH_REGEX.match(header): # Preconditions are handled elsewhere, but allow these headers through. continue elif GOOG_METAGENERATION_MATCH_REGEX.match(header): # Preconditions are handled elsewhere, but allow these headers through. continue else: custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header) custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header) custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header) header_key = None if custom_goog_metadata_match: header_key = custom_goog_metadata_match.group('header_key') elif custom_amz_metadata_match: header_key = custom_amz_metadata_match.group('header_key') elif custom_amz_header_match: # If we got here we are guaranteed by the prior statement that this is # not an x-amz-meta- header. header_key = (S3_HEADER_PREFIX + custom_amz_header_match.group('header_key')) if header_key: if header_key.lower() == 'x-goog-content-language': # Work around content-language being inserted into custom metadata. continue if not obj_metadata.metadata: obj_metadata.metadata = apitools_messages.Object.MetadataValue() if not obj_metadata.metadata.additionalProperties: obj_metadata.metadata.additionalProperties = [] obj_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=header_key, value=value)) return obj_metadata def HeadersFromObjectMetadata(dst_obj_metadata, provider): """Creates a header dictionary based on existing object metadata. Args: dst_obj_metadata: Object metadata to create the headers from. provider: Provider string ('gs' or 's3'). Returns: Headers dictionary. """ headers = {} if not dst_obj_metadata: return # Metadata values of '' mean suppress/remove this header. if dst_obj_metadata.cacheControl is not None: if not dst_obj_metadata.cacheControl: headers['cache-control'] = None else: headers['cache-control'] = dst_obj_metadata.cacheControl.strip() if dst_obj_metadata.contentDisposition: if not dst_obj_metadata.contentDisposition: headers['content-disposition'] = None else: headers['content-disposition'] = ( dst_obj_metadata.contentDisposition.strip()) if dst_obj_metadata.contentEncoding: if not dst_obj_metadata.contentEncoding: headers['content-encoding'] = None else: headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip() if dst_obj_metadata.contentLanguage: if not dst_obj_metadata.contentLanguage: headers['content-language'] = None else: headers['content-language'] = dst_obj_metadata.contentLanguage.strip() if dst_obj_metadata.md5Hash: if not dst_obj_metadata.md5Hash: headers['Content-MD5'] = None else: headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip() if dst_obj_metadata.contentType is not None: if not dst_obj_metadata.contentType: headers['content-type'] = None else: headers['content-type'] = dst_obj_metadata.contentType.strip() if dst_obj_metadata.customTime is not None: if not dst_obj_metadata.customTime: headers['custom-time'] = None else: headers['custom-time'] = dst_obj_metadata.customTime.strip() if dst_obj_metadata.storageClass: header_name = 'storage-class' if provider == 'gs': header_name = 'x-goog-' + header_name elif provider == 's3': header_name = 'x-amz-' + header_name else: raise ArgumentException('Invalid provider specified: %s' % provider) headers[header_name] = dst_obj_metadata.storageClass.strip() if (dst_obj_metadata.metadata and dst_obj_metadata.metadata.additionalProperties): for additional_property in dst_obj_metadata.metadata.additionalProperties: # Work around content-language being inserted into custom metadata by # the XML API. if additional_property.key == 'content-language': continue # Don't translate special metadata markers. if additional_property.key in S3_MARKER_GUIDS: continue if provider == 'gs': header_name = 'x-goog-meta-' + additional_property.key elif provider == 's3': if additional_property.key.startswith(S3_HEADER_PREFIX): header_name = ('x-amz-' + additional_property.key[len(S3_HEADER_PREFIX):]) else: header_name = 'x-amz-meta-' + additional_property.key else: raise ArgumentException('Invalid provider specified: %s' % provider) if (additional_property.value is not None and not additional_property.value): headers[header_name] = None else: headers[header_name] = additional_property.value return headers def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False): """Copies metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. """ if override or not dst_obj_metadata.cacheControl: dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl if override or not dst_obj_metadata.contentDisposition: dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition if override or not dst_obj_metadata.contentEncoding: dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding if override or not dst_obj_metadata.contentLanguage: dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage if override or not dst_obj_metadata.contentType: dst_obj_metadata.contentType = src_obj_metadata.contentType if override or not dst_obj_metadata.customTime: dst_obj_metadata.customTime = src_obj_metadata.customTime if override or not dst_obj_metadata.md5Hash: dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash CopyCustomMetadata(src_obj_metadata, dst_obj_metadata, override=override) def CopyCustomMetadata(src_obj_metadata, dst_obj_metadata, override=False): """Copies custom metadata from src_obj_metadata to dst_obj_metadata. Args: src_obj_metadata: Metadata from source object. dst_obj_metadata: Initialized metadata for destination object. override: If true, will overwrite metadata in destination object. If false, only writes metadata for values that don't already exist. """ # TODO: Apitools should ideally treat metadata like a real dictionary instead # of a list of key/value pairs (with an O(N^2) lookup). In practice the # number of values is typically small enough not to matter. # Work around this by creating our own dictionary. if (src_obj_metadata.metadata and src_obj_metadata.metadata.additionalProperties): if not dst_obj_metadata.metadata: dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue() if not dst_obj_metadata.metadata.additionalProperties: dst_obj_metadata.metadata.additionalProperties = [] dst_metadata_dict = {} for dst_prop in dst_obj_metadata.metadata.additionalProperties: dst_metadata_dict[dst_prop.key] = dst_prop.value for src_prop in src_obj_metadata.metadata.additionalProperties: if src_prop.key in dst_metadata_dict: if override: # Metadata values of '' mean suppress/remove this header. if src_prop.value is not None and not src_prop.value: dst_metadata_dict[src_prop.key] = None else: dst_metadata_dict[src_prop.key] = src_prop.value elif src_prop.value != '': # pylint: disable=g-explicit-bool-comparison # Don't propagate '' value since that means to remove the header. dst_metadata_dict[src_prop.key] = src_prop.value # Rewrite the list with our updated dict. dst_obj_metadata.metadata.additionalProperties = [] for k, v in six.iteritems(dst_metadata_dict): dst_obj_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, value=v)) def PreconditionsFromHeaders(headers): """Creates bucket or object preconditions acccording to the provided headers. Args: headers: Dict of headers passed via gsutil -h Returns: gsutil Cloud API Preconditions object fields populated from headers, or None if no precondition headers are present. """ return_preconditions = Preconditions() try: for header, value in headers.items(): if GOOG_GENERATION_MATCH_REGEX.match(header): return_preconditions.gen_match = long(value) if GOOG_METAGENERATION_MATCH_REGEX.match(header): return_preconditions.meta_gen_match = long(value) except ValueError as _: raise ArgumentException('Invalid precondition header specified. ' 'x-goog-if-generation-match and ' 'x-goog-if-metageneration match must be specified ' 'with a positive integer value.') return return_preconditions def CreateNotFoundExceptionForObjectWrite(dst_provider, dst_bucket_name, src_provider=None, src_bucket_name=None, src_object_name=None, src_generation=None): """Creates a NotFoundException for an object upload or copy. This is necessary because 404s don't necessarily specify which resource does not exist. Args: dst_provider: String abbreviation of destination provider, e.g., 'gs'. dst_bucket_name: Destination bucket name for the write operation. src_provider: String abbreviation of source provider, i.e. 'gs', if any. src_bucket_name: Source bucket name, if any (for the copy case). src_object_name: Source object name, if any (for the copy case). src_generation: Source object generation, if any (for the copy case). Returns: NotFoundException with appropriate message. """ dst_url_string = '%s://%s' % (dst_provider, dst_bucket_name) if src_bucket_name and src_object_name: src_url_string = '%s://%s/%s' % (src_provider, src_bucket_name, src_object_name) if src_generation: src_url_string += '#%s' % str(src_generation) return NotFoundException( 'The source object %s or the destination bucket %s does not exist.' % (src_url_string, dst_url_string)) return NotFoundException( 'The destination bucket %s does not exist or the write to the ' 'destination must be restarted' % dst_url_string) def CreateBucketNotFoundException(code, provider, bucket_name): return BucketNotFoundException('%s://%s bucket does not exist.' % (provider, bucket_name), bucket_name, status=code) def CreateObjectNotFoundException(code, provider, bucket_name, object_name, generation=None): uri_string = '%s://%s/%s' % (provider, bucket_name, object_name) if generation: uri_string += '#%s' % str(generation) return NotFoundException('%s does not exist.' % uri_string, status=code) def CheckForXmlConfigurationAndRaise(config_type_string, json_txt): """Checks a JSON parse exception for provided XML configuration.""" try: xml.etree.ElementTree.fromstring(str(json_txt)) raise ArgumentException('\n'.join( textwrap.wrap( 'XML {0} data provided; Google Cloud Storage {0} configuration ' 'now uses JSON format. To convert your {0}, set the desired XML ' 'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then ' 'use \'gsutil {1} get ...\' with gsutil version 4 or greater to ' 'get the corresponding JSON {0}.'.format( config_type_string, config_type_string.lower())))) except XmlParseError: pass raise ArgumentException('JSON %s data could not be loaded ' 'from: %s' % (config_type_string, json_txt)) class LifecycleTranslation(object): """Functions for converting between various lifecycle formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. """ @classmethod def BotoLifecycleFromMessage(cls, lifecycle_message): """Translates an apitools message to a boto lifecycle object.""" boto_lifecycle = boto.gs.lifecycle.LifecycleConfig() if lifecycle_message: for rule_message in lifecycle_message.rule: boto_rule = boto.gs.lifecycle.Rule() if rule_message.action and rule_message.action.type: if rule_message.action.type.lower() == 'delete': boto_rule.action = boto.gs.lifecycle.DELETE elif rule_message.action.type.lower() == 'setstorageclass': boto_rule.action = boto.gs.lifecycle.SET_STORAGE_CLASS boto_rule.action_text = rule_message.action.storageClass if rule_message.condition: if rule_message.condition.age is not None: boto_rule.conditions[boto.gs.lifecycle.AGE] = (str( rule_message.condition.age)) if rule_message.condition.createdBefore: boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = (str( rule_message.condition.createdBefore)) if rule_message.condition.isLive is not None: boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = ( # Note that the GCS XML API only accepts "false" or "true" # in all lower case. str(rule_message.condition.isLive).lower()) if rule_message.condition.matchesStorageClass: boto_rule.conditions[boto.gs.lifecycle.MATCHES_STORAGE_CLASS] = [ str(sc) for sc in rule_message.condition.matchesStorageClass ] if rule_message.condition.numNewerVersions is not None: boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = (str( rule_message.condition.numNewerVersions)) boto_lifecycle.append(boto_rule) return boto_lifecycle @classmethod def BotoLifecycleToMessage(cls, boto_lifecycle): """Translates a boto lifecycle object to an apitools message.""" lifecycle_message = None if boto_lifecycle: lifecycle_message = apitools_messages.Bucket.LifecycleValue() for boto_rule in boto_lifecycle: lifecycle_rule = ( apitools_messages.Bucket.LifecycleValue.RuleValueListEntry()) lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ConditionValue()) if boto_rule.action: if boto_rule.action == boto.gs.lifecycle.DELETE: lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ActionValue( type='Delete')) elif boto_rule.action == boto.gs.lifecycle.SET_STORAGE_CLASS: lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue. RuleValueListEntry.ActionValue( type='SetStorageClass', storageClass=boto_rule.action_text)) if boto.gs.lifecycle.AGE in boto_rule.conditions: lifecycle_rule.condition.age = int( boto_rule.conditions[boto.gs.lifecycle.AGE]) if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions: lifecycle_rule.condition.createdBefore = ( LifecycleTranslation.TranslateBotoLifecycleTimestamp( boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE])) if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions: boto_is_live_str = ( boto_rule.conditions[boto.gs.lifecycle.IS_LIVE].lower()) if boto_is_live_str == 'true': lifecycle_rule.condition.isLive = True elif boto_is_live_str == 'false': lifecycle_rule.condition.isLive = False else: raise CommandException( 'Got an invalid Boto value for IsLive condition ("%s"), ' 'expected "true" or "false".' % boto_rule.conditions[boto.gs.lifecycle.IS_LIVE]) if boto.gs.lifecycle.MATCHES_STORAGE_CLASS in boto_rule.conditions: for storage_class in ( boto_rule.conditions[boto.gs.lifecycle.MATCHES_STORAGE_CLASS]): lifecycle_rule.condition.matchesStorageClass.append(storage_class) if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions: lifecycle_rule.condition.numNewerVersions = int( boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS]) lifecycle_message.rule.append(lifecycle_rule) return lifecycle_message @classmethod def JsonLifecycleFromMessage(cls, lifecycle_message): """Translates an apitools message to lifecycle JSON.""" return str(encoding.MessageToJson(lifecycle_message)) + '\n' @classmethod def JsonLifecycleToMessage(cls, json_txt): """Translates lifecycle JSON to an apitools message.""" try: deserialized_lifecycle = json.loads(json_txt) # If lifecycle JSON is the in the following format # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key # and reduce it to the following format # {'rule': ... if 'lifecycle' in deserialized_lifecycle: deserialized_lifecycle = deserialized_lifecycle['lifecycle'] lifecycle = encoding.DictToMessage( deserialized_lifecycle or {}, apitools_messages.Bucket.LifecycleValue) return lifecycle except ValueError: CheckForXmlConfigurationAndRaise('lifecycle', json_txt) @classmethod def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime): """Parses the timestamp from the boto lifecycle into a datetime object.""" return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date() class CorsTranslation(object): """Functions for converting between various CORS formats. This class handles conversation to and from Boto Cors objects, JSON text, and apitools Message objects. """ @classmethod def BotoCorsFromMessage(cls, cors_message): """Translates an apitools message to a boto Cors object.""" cors = boto.gs.cors.Cors() cors.cors = [] for collection_message in cors_message: collection_elements = [] if collection_message.maxAgeSeconds: collection_elements.append( (boto.gs.cors.MAXAGESEC, str(collection_message.maxAgeSeconds))) if collection_message.method: method_elements = [] for method in collection_message.method: method_elements.append((boto.gs.cors.METHOD, method)) collection_elements.append((boto.gs.cors.METHODS, method_elements)) if collection_message.origin: origin_elements = [] for origin in collection_message.origin: origin_elements.append((boto.gs.cors.ORIGIN, origin)) collection_elements.append((boto.gs.cors.ORIGINS, origin_elements)) if collection_message.responseHeader: header_elements = [] for header in collection_message.responseHeader: header_elements.append((boto.gs.cors.HEADER, header)) collection_elements.append((boto.gs.cors.HEADERS, header_elements)) cors.cors.append(collection_elements) return cors @classmethod def BotoCorsToMessage(cls, boto_cors): """Translates a boto Cors object to an apitools message.""" message_cors = [] if boto_cors.cors: for cors_collection in boto_cors.cors: if cors_collection: collection_message = apitools_messages.Bucket.CorsValueListEntry() for element_tuple in cors_collection: if element_tuple[0] == boto.gs.cors.MAXAGESEC: collection_message.maxAgeSeconds = int(element_tuple[1]) if element_tuple[0] == boto.gs.cors.METHODS: for method_tuple in element_tuple[1]: collection_message.method.append(method_tuple[1]) if element_tuple[0] == boto.gs.cors.ORIGINS: for origin_tuple in element_tuple[1]: collection_message.origin.append(origin_tuple[1]) if element_tuple[0] == boto.gs.cors.HEADERS: for header_tuple in element_tuple[1]: collection_message.responseHeader.append(header_tuple[1]) message_cors.append(collection_message) return message_cors @classmethod def JsonCorsToMessageEntries(cls, json_cors): """Translates CORS JSON to an apitools message. Args: json_cors: JSON string representing CORS configuration. Raises: ArgumentException on invalid CORS JSON data. Returns: List of apitools Bucket.CorsValueListEntry. An empty list represents no CORS configuration. """ deserialized_cors = None try: deserialized_cors = json.loads(json_cors) except ValueError: CheckForXmlConfigurationAndRaise('CORS', json_cors) if not isinstance(deserialized_cors, list): raise ArgumentException( 'CORS JSON should be formatted as a list containing one or more JSON ' 'objects.\nSee "gsutil help cors".') cors = [] for cors_entry in deserialized_cors: cors.append( encoding.DictToMessage(cors_entry, apitools_messages.Bucket.CorsValueListEntry)) return cors @classmethod def MessageEntriesToJson(cls, cors_message): """Translates an apitools message to CORS JSON.""" json_text = '' # Because CORS is a MessageField, serialize/deserialize as JSON list. json_text += '[' printed_one = False for cors_entry in cors_message: if printed_one: json_text += ',' else: printed_one = True json_text += encoding.MessageToJson(cors_entry) json_text += ']\n' return json_text def S3MarkerAclFromObjectMetadata(object_metadata): """Retrieves GUID-marked S3 ACL from object metadata, if present. Args: object_metadata: Object metadata to check. Returns: S3 ACL text, if present, None otherwise. """ if (object_metadata and object_metadata.metadata and object_metadata.metadata.additionalProperties): for prop in object_metadata.metadata.additionalProperties: if prop.key == S3_ACL_MARKER_GUID: return prop.value def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text): """Adds a GUID-marked S3 ACL to the object metadata. Args: object_metadata: Object metadata to add the acl to. acl_text: S3 ACL text to add. """ if not object_metadata.metadata: object_metadata.metadata = apitools_messages.Object.MetadataValue() if not object_metadata.metadata.additionalProperties: object_metadata.metadata.additionalProperties = [] object_metadata.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=S3_ACL_MARKER_GUID, value=acl_text)) def UnaryDictToXml(message): """Generates XML representation of a nested dict. This dict contains exactly one top-level entry and an arbitrary number of 2nd-level entries, e.g. capturing a WebsiteConfiguration message. Args: message: The dict encoding the message. Returns: XML string representation of the input dict. Raises: Exception: if dict contains more than one top-level entry. """ if len(message) != 1: raise Exception('Expected dict of size 1, got size %d' % len(message)) name, content = message.items()[0] element_type = xml.etree.ElementTree.Element(name) for element_property, value in sorted(content.items()): node = xml.etree.ElementTree.SubElement(element_type, element_property) node.text = value return xml.etree.ElementTree.tostring(element_type) class LabelTranslation(object): """Functions for converting between various Label(JSON)/Tags(XML) formats. This class handles conversion to and from Boto Tags objects, JSON text, and apitools LabelsValue message objects. """ @classmethod def BotoTagsToMessage(cls, tags): label_dict = {} for tag_set in tags: label_dict.update(dict((i.key, i.value) for i in tag_set)) return cls.DictToMessage(label_dict) if label_dict else None @classmethod def BotoTagsFromMessage(cls, message): label_dict = json.loads(cls.JsonFromMessage(message)) tag_set = TagSet() for key, value in six.iteritems(label_dict): if value: # Skip values which may be set to None. tag_set.add_tag(key, value) tags = Tags() tags.add_tag_set(tag_set) return tags @classmethod def JsonFromMessage(cls, message, pretty_print=False): json_str = encoding.MessageToJson(message) if pretty_print: return json.dumps(json.loads(json_str), sort_keys=True, indent=2, separators=(',', ': ')) return json_str @classmethod def DictToMessage(cls, label_dict): return encoding.DictToMessage(label_dict, apitools_messages.Bucket.LabelsValue) class AclTranslation(object): """Functions for converting between various ACL formats. This class handles conversion to and from Boto ACL objects, JSON text, and apitools Message objects. """ JSON_TO_XML_ROLES = { 'READER': 'READ', 'WRITER': 'WRITE', 'OWNER': 'FULL_CONTROL', } XML_TO_JSON_ROLES = { 'READ': 'READER', 'WRITE': 'WRITER', 'FULL_CONTROL': 'OWNER', } @classmethod def BotoAclFromJson(cls, acl_json): acl = ACL() acl.parent = None acl.entries = cls.BotoEntriesFromJson(acl_json, acl) return acl @classmethod # acl_message is a list of messages, either object or bucketaccesscontrol def BotoAclFromMessage(cls, acl_message): acl_dicts = [] for message in acl_message: if message == PRIVATE_DEFAULT_OBJ_ACL: # Sentinel value indicating acl_dicts should be an empty list to create # a private (no entries) default object ACL. break acl_dicts.append(encoding.MessageToDict(message)) return cls.BotoAclFromJson(acl_dicts) @classmethod def BotoAclToJson(cls, acl): if hasattr(acl, 'entries'): return cls.BotoEntriesToJson(acl.entries) return [] @classmethod def BotoObjectAclToMessage(cls, acl): for entry in cls.BotoAclToJson(acl): message = encoding.DictToMessage(entry, apitools_messages.ObjectAccessControl) message.kind = 'storage#objectAccessControl' yield message @classmethod def BotoBucketAclToMessage(cls, acl): for entry in cls.BotoAclToJson(acl): message = encoding.DictToMessage(entry, apitools_messages.BucketAccessControl) message.kind = 'storage#bucketAccessControl' yield message @classmethod def BotoEntriesFromJson(cls, acl_json, parent): entries = Entries(parent) entries.parent = parent entries.entry_list = [ cls.BotoEntryFromJson(entry_json) for entry_json in acl_json ] return entries @classmethod def BotoEntriesToJson(cls, entries): return [cls.BotoEntryToJson(entry) for entry in entries.entry_list] @classmethod def BotoEntryFromJson(cls, entry_json): """Converts a JSON entry into a Boto ACL entry.""" entity = entry_json['entity'] permission = cls.JSON_TO_XML_ROLES[entry_json['role']] if entity.lower() == ALL_USERS.lower(): return Entry(type=ALL_USERS, permission=permission) elif entity.lower() == ALL_AUTHENTICATED_USERS.lower(): return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission) elif entity.startswith('project'): raise CommandException('XML API does not support project scopes, ' 'cannot translate ACL.') elif 'email' in entry_json: if entity.startswith('user'): scope_type = USER_BY_EMAIL elif entity.startswith('group'): scope_type = GROUP_BY_EMAIL return Entry(type=scope_type, email_address=entry_json['email'], permission=permission) elif 'entityId' in entry_json: if entity.startswith('user'): scope_type = USER_BY_ID elif entity.startswith('group'): scope_type = GROUP_BY_ID return Entry(type=scope_type, id=entry_json['entityId'], permission=permission) elif 'domain' in entry_json: if entity.startswith('domain'): scope_type = GROUP_BY_DOMAIN return Entry(type=scope_type, domain=entry_json['domain'], permission=permission) raise CommandException('Failed to translate JSON ACL to XML.') @classmethod def BotoEntryToJson(cls, entry): """Converts a Boto ACL entry to a valid JSON dictionary.""" acl_entry_json = {} # JSON API documentation uses camel case. scope_type_lower = entry.scope.type.lower() if scope_type_lower == ALL_USERS.lower(): acl_entry_json['entity'] = 'allUsers' elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower(): acl_entry_json['entity'] = 'allAuthenticatedUsers' elif scope_type_lower == USER_BY_EMAIL.lower(): acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address acl_entry_json['email'] = entry.scope.email_address elif scope_type_lower == USER_BY_ID.lower(): acl_entry_json['entity'] = 'user-%s' % entry.scope.id acl_entry_json['entityId'] = entry.scope.id elif scope_type_lower == GROUP_BY_EMAIL.lower(): acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address acl_entry_json['email'] = entry.scope.email_address elif scope_type_lower == GROUP_BY_ID.lower(): acl_entry_json['entity'] = 'group-%s' % entry.scope.id acl_entry_json['entityId'] = entry.scope.id elif scope_type_lower == GROUP_BY_DOMAIN.lower(): acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain acl_entry_json['domain'] = entry.scope.domain else: raise ArgumentException('ACL contains invalid scope type: %s' % scope_type_lower) acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission] return acl_entry_json @classmethod def JsonToMessage(cls, json_data, message_type): """Converts the input JSON data into list of Object/BucketAccessControls. Args: json_data: String of JSON to convert. message_type: Which type of access control entries to return, either ObjectAccessControl or BucketAccessControl. Raises: ArgumentException on invalid JSON data. Returns: List of ObjectAccessControl or BucketAccessControl elements. """ try: deserialized_acl = json.loads(json_data) acl = [] for acl_entry in deserialized_acl: acl.append(encoding.DictToMessage(acl_entry, message_type)) return acl except ValueError: CheckForXmlConfigurationAndRaise('ACL', json_data) @classmethod def JsonFromMessage(cls, acl): """Strips unnecessary fields from an ACL message and returns valid JSON. Args: acl: iterable ObjectAccessControl or BucketAccessControl Returns: ACL JSON string. """ serializable_acl = [] if acl is not None: for acl_entry in acl: if acl_entry.kind == 'storage#objectAccessControl': acl_entry.object = None acl_entry.generation = None acl_entry.kind = None acl_entry.bucket = None acl_entry.id = None acl_entry.selfLink = None acl_entry.etag = None serializable_acl.append(encoding.MessageToDict(acl_entry)) return json.dumps(serializable_acl, sort_keys=True, indent=2, separators=(',', ': '))	38.41 KB	2025-06-25 08:19
📄 unit_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods that calculate, convert, and simplify units.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import math import re import six if six.PY3: long = int # Binary exponentiation strings. _EXP_STRINGS = [ (0, 'B', 'bit'), (10, 'KiB', 'Kibit', 'K'), (20, 'MiB', 'Mibit', 'M'), (30, 'GiB', 'Gibit', 'G'), (40, 'TiB', 'Tibit', 'T'), (50, 'PiB', 'Pibit', 'P'), (60, 'EiB', 'Eibit', 'E'), ] _EXP_TEN_STRING = [ (3, 'k'), (6, 'm'), (9, 'b'), (12, 't'), (15, 'q'), ] # Define this method before constants below, as some call it to init themselves. def _GenerateSuffixRegex(): """Creates a suffix regex for human-readable byte counts.""" human_bytes_re = r'(?P<num>\d\.\d+\|\d+)\s(?P<suffix>%s)?' suffixes = [] suffix_to_si = {} for i, si in enumerate(_EXP_STRINGS): si_suffixes = [s.lower() for s in list(si)[1:]] for suffix in si_suffixes: suffix_to_si[suffix] = i suffixes.extend(si_suffixes) human_bytes_re %= '\|'.join(suffixes) matcher = re.compile(human_bytes_re) return suffix_to_si, matcher # TODO: These should include the unit in the name, e.g. BYTES_PER_KIB, or # BYTES_IN_ONE_KIB. ONE_KIB = 1024 ONE_MIB = 1024 * ONE_KIB ONE_GIB = 1024 * ONE_MIB # TODO: Remove 2, 8, 10 MIB vars. TWO_MIB = 2 * ONE_MIB EIGHT_MIB = 8 * ONE_MIB TEN_MIB = 10 * ONE_MIB SECONDS_PER_DAY = long(60 * 60 * 24) SUFFIX_TO_SI, MATCH_HUMAN_BYTES = _GenerateSuffixRegex() def _RoundToNearestExponent(num): i = 0 while i + 1 < len(_EXP_STRINGS) and num >= (2_EXP_STRINGS[i + 1][0]): i += 1 return i, round(float(num) / 2.0_EXP_STRINGS[i][0], 2) def CalculateThroughput(total_bytes_transferred, total_elapsed_time): """Calculates throughput and checks for a small total_elapsed_time. Args: total_bytes_transferred: Total bytes transferred in a period of time. total_elapsed_time: The amount of time elapsed in seconds. Returns: The throughput as a float. """ if total_elapsed_time < 0.01: total_elapsed_time = 0.01 return float(total_bytes_transferred) / float(total_elapsed_time) def DecimalShort(num): """Creates a shorter string version for a given number of objects. Args: num: The number of objects to be shortened. Returns: shortened string version for this number. It takes the largest scale (thousand, million or billion) smaller than the number and divides it by that scale, indicated by a suffix with one decimal place. This will thus create a string of at most 6 characters, assuming num < 10^18. Example: 123456789 => 123.4m """ for divisor_exp, suffix in reversed(_EXP_TEN_STRING): if num >= 10divisor_exp: quotient = '%.1lf' % (float(num) / 10divisor_exp) return quotient + suffix return str(num) def DivideAndCeil(dividend, divisor): """Returns ceil(dividend / divisor). Takes care to avoid the pitfalls of floating point arithmetic that could otherwise yield the wrong result for large numbers. Args: dividend: Dividend for the operation. divisor: Divisor for the operation. Returns: Quotient. """ quotient = dividend // divisor if (dividend % divisor) != 0: quotient += 1 return quotient def HumanReadableToBytes(human_string): """Tries to convert a human-readable string to a number of bytes. Args: human_string: A string supplied by user, e.g. '1M', '3 GiB'. Returns: An integer containing the number of bytes. Raises: ValueError: on an invalid string. """ human_string = human_string.lower() m = MATCH_HUMAN_BYTES.match(human_string) if m: num = float(m.group('num')) if m.group('suffix'): power = _EXP_STRINGS[SUFFIX_TO_SI[m.group('suffix')]][0] num = (2.0power) num = int(round(num)) return num raise ValueError('Invalid byte string specified: %s' % human_string) def HumanReadableWithDecimalPlaces(number, decimal_places=1): """Creates a human readable format for bytes with fixed decimal places. Args: number: The number of bytes. decimal_places: The number of decimal places. Returns: String representing a readable format for number with decimal_places decimal places. """ number_format = MakeHumanReadable(number).split() num = str(int(round(10decimal_places float(number_format[0])))) if num == '0': number_format[0] = ('0' + (('.' + ('0' * decimal_places)) if decimal_places else '')) else: num_length = len(num) if decimal_places: num = (num[:num_length - decimal_places] + '.' + num[num_length - decimal_places:]) number_format[0] = num return ' '.join(number_format) def MakeBitsHumanReadable(num): """Generates human readable string for a number of bits. Args: num: The number, in bits. Returns: A string form of the number using bit size abbreviations (kbit, Mbit, etc.) """ i, rounded_val = _RoundToNearestExponent(num) return '%g %s' % (rounded_val, _EXP_STRINGS[i][2]) def MakeHumanReadable(num): """Generates human readable string for a number of bytes. Args: num: The number, in bytes. Returns: A string form of the number using size abbreviations (KiB, MiB, etc.). """ i, rounded_val = _RoundToNearestExponent(num) return '%g %s' % (rounded_val, _EXP_STRINGS[i][1]) def Percentile(values, percent, key=lambda x: x): """Find the percentile of a list of values. Taken from: http://code.activestate.com/recipes/511478/ Args: values: a list of numeric values. Note that the values MUST BE already sorted. percent: a float value from 0.0 to 1.0. key: optional key function to compute value from each element of the list of values. Returns: The percentile of the values. """ if not values: return None k = (len(values) - 1) * percent f = math.floor(k) c = math.ceil(k) if f == c: return key(values[int(k)]) d0 = key(values[int(f)]) * (c - k) d1 = key(values[int(c)]) * (k - f) return d0 + d1 def PrettyTime(remaining_time): """Creates a standard version for a given remaining time in seconds. Created over using strftime because strftime seems to be more suitable for a datetime object, rather than just a number of seconds remaining. Args: remaining_time: The number of seconds remaining as a float, or a string/None value indicating time was not correctly calculated. Returns: if remaining_time is a valid float, %H:%M:%D time remaining format with the nearest integer from remaining_time (%H might be higher than 23). Else, it returns the same message it received. """ remaining_time = int(round(remaining_time)) hours = remaining_time // 3600 if hours >= 100: # Too large to display with precision of minutes and seconds. # If over 1000, saying 999+ hours should be enough. return '%d+ hrs' % min(hours, 999) remaining_time -= (3600 * hours) minutes = remaining_time // 60 remaining_time -= (60 * minutes) seconds = remaining_time return (str('%02d' % hours) + ':' + str('%02d' % minutes) + ':' + str('%02d' % seconds))	7.73 KB	2025-06-25 08:19
📄 update_util.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared utility methods for the update command and its tests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import re import textwrap import sys import gslib from gslib.utils.system_util import IS_OSX from gslib.exception import CommandException from gslib.storage_url import StorageUrlFromString from gslib.utils.constants import GSUTIL_PUB_TARBALL from gslib.utils.constants import GSUTIL_PUB_TARBALL_PY2 # This function used to belong inside of update.py. However, it needed to be # moved here due to compatibility issues with Travis CI, because update.py is # not included with PyPI installations. def DisallowUpdateIfDataInGsutilDir(directory=gslib.GSUTIL_DIR): """Disallows the update command if files not in the gsutil distro are found. This prevents users from losing data if they are in the habit of running gsutil from the gsutil directory and leaving data in that directory. This will also detect someone attempting to run gsutil update from a git repo, since the top-level directory will contain git files and dirs (like .git) that are not distributed with gsutil. Args: directory: (str) The directory to use this functionality on. Raises: CommandException: if files other than those distributed with gsutil found. """ # Manifest includes recursive-includes of gslib. Directly add # those to the list here so we will skip them in os.listdir() loop without # having to build deeper handling of the MANIFEST file here. Also include # 'third_party', which isn't present in manifest but gets added to the # gsutil distro by the gsutil submodule configuration; and the MANIFEST.in # and CHANGES.md files. manifest_lines = ['MANIFEST.in', 'third_party'] try: with open(os.path.join(directory, 'MANIFEST.in'), 'r') as fp: for line in fp: if line.startswith('include '): manifest_lines.append(line.split()[-1]) elif re.match(r'recursive-include \w+ \*', line): manifest_lines.append(line.split()[1]) except IOError: logging.getLogger().warn( 'MANIFEST.in not found in %s.\nSkipping user data ' 'check.\n', directory) return # Look just at top-level directory. We don't try to catch data dropped into # subdirs (like gslib) because that would require deeper parsing of # MANFFEST.in, and most users who drop data into gsutil dir do so at the top # level directory. addl_excludes = ( '.coverage', '.DS_Store', '.github', '.style.yapf', '.yapfignore', '__pycache__', '.github', ) for filename in os.listdir(directory): if filename.endswith('.pyc') or filename in addl_excludes: continue if filename not in manifest_lines: raise CommandException('\n'.join( textwrap.wrap( 'A file (%s) that is not distributed with gsutil was found in ' 'the gsutil directory. The update command cannot run with user ' 'data in the gsutil directory.' % os.path.join(gslib.GSUTIL_DIR, filename)))) def LookUpGsutilVersion(gsutil_api, url_str): """Looks up the gsutil version of the specified gsutil tarball URL. Version is specified in the metadata field set on that object. Args: gsutil_api: gsutil Cloud API to use when retrieving gsutil tarball. url_str: tarball URL to retrieve (such as 'gs://pub/gsutil.tar.gz'). Returns: Version string if URL is a cloud URL containing x-goog-meta-gsutil-version metadata, else None. """ url = StorageUrlFromString(url_str) if url.IsCloudUrl(): obj = gsutil_api.GetObjectMetadata(url.bucket_name, url.object_name, provider=url.scheme, fields=['metadata']) if obj.metadata and obj.metadata.additionalProperties: for prop in obj.metadata.additionalProperties: if prop.key == 'gsutil_version': return prop.value def GsutilPubTarball(): """Returns the appropriate gsutil pub tarball based on the Python version. Returns: The storage_uri of the appropriate pub tarball. """ if sys.version_info.major == 2: return GSUTIL_PUB_TARBALL_PY2 return GSUTIL_PUB_TARBALL	4.88 KB	2025-06-25 08:19
📄 user_agent_helper.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains helper for appending user agent information.""" import six import sys import uuid import gslib from gslib.utils import system_util from gslib.storage_url import StorageUrlFromString from gslib.exception import InvalidUrlError def GetUserAgent(args, metrics_off=True): """Using the command arguments return a suffix for the UserAgent string. Args: args: str[], parsed set of arguments entered in the CLI. metrics_off: boolean, whether the MetricsCollector is disabled. Returns: str, A string value that can be appended to an existing UserAgent. """ user_agent = ' gsutil/%s' % gslib.VERSION user_agent += ' (%s)' % sys.platform user_agent += ' analytics/%s' % ('disabled' if metrics_off else 'enabled') user_agent += ' interactive/%s' % system_util.IsRunningInteractively() user_agent += ' invocation-id/%s' % uuid.uuid4().hex if len(args) > 0: user_agent += ' command/%s' % args[0] if len(args) > 2: if args[0] in ['cp', 'mv', 'rsync']: # Any cp, mv or rsync commands that use daisy chain mode should be noted # as that represents a unique use case that may be better served by the # storage transfer service. try: src = StorageUrlFromString(six.ensure_text(args[-2])) dst = StorageUrlFromString(six.ensure_text(args[-1])) if src.IsCloudUrl() and dst.IsCloudUrl() and src.scheme != dst.scheme: user_agent += '-DaisyChain' except InvalidUrlError: pass elif args[0] == 'rewrite': if '-k' in args: # Rewrite encryption key. user_agent += '-k' if '-s' in args: # Rewrite storage class. user_agent += '-s' if '-m' in args: # Use of multi processing user_agent += '-m' if system_util.InvokedViaCloudSdk(): user_agent += ' google-cloud-sdk' if system_util.CloudSdkVersion(): user_agent += '/%s' % system_util.CloudSdkVersion() return user_agent	2.55 KB	2025-06-25 08:19
📄 version_check.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Utility functions to ensure the correct version of Python is used.""" import sys from sys import version_info # Key: Int, supported Python major version # Value: Int, min supported Python minor version # We currently support Python >=3.5 MIN_SUPPORTED_PYTHON_VERSION = {3: 5} def check_python_version_support(): """Return an exception if running in an unsupported version of Python. This function compares the running version of cPython and against the list of supported python version. If the running version is less than any of the supported versions, return a Tuple of (False, Str(error message)) for the caller to handle. Minor versions of Python greater than those listed in the supported versions are allowed. Args: None Returns: Tuple(Boolean, String) A Tuple containing a Boolean and a String. The boolean represents if the version is supported, and the String will either be empty, or contain an error message. """ major = sys.version_info.major minor = sys.version_info.minor if major not in MIN_SUPPORTED_PYTHON_VERSION: return (False, 'Gsutil does not support running under Python{major}'.format( major=major)) if minor < MIN_SUPPORTED_PYTHON_VERSION[major]: lowest_minor = MIN_SUPPORTED_PYTHON_VERSION[major] return ( False, 'For Python{major}, gsutil requires Python{major}.{lowest_minor}+, but ' 'you are using Python{major}.{minor}'.format(major=major, minor=minor, lowest_minor=lowest_minor)) return (True, '')	2.21 KB	2025-06-25 08:19
📄 wrapped_credentials.py # -- coding: utf-8 -- # Copyright 2022 Google LLC. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Classes and functions to allow google.auth credentials to be used within oauth2client. In particular, the External Account credentials don't have an equivalent in oauth2client, so we create helper methods to allow variants of this particular class to be used in oauth2client workflows. """ import copy import datetime import io import json from google.auth import aws from google.auth import credentials from google.auth import exceptions from google.auth import external_account from google.auth import external_account_authorized_user from google.auth import identity_pool from google.auth import pluggable from google.auth.transport import requests from gslib.utils import constants import oauth2client DEFAULT_SCOPES = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.CLOUD_PLATFORM_READ_ONLY, constants.Scopes.FULL_CONTROL, constants.Scopes.READ_ONLY, constants.Scopes.READ_WRITE, ] class WrappedCredentials(oauth2client.client.OAuth2Credentials): """A utility class to use Google Auth credentials in place of oauth2client credentials. """ NON_SERIALIZED_MEMBERS = frozenset( list(oauth2client.client.OAuth2Credentials.NON_SERIALIZED_MEMBERS) + ['_base']) def __init__(self, base_creds): """Initializes oauth2client credentials based on underlying Google Auth credentials. Args: external_account_creds: subclass of google.auth.external_account.Credentials """ self._base = base_creds if isinstance(base_creds, external_account.Credentials): client_id = self._base._audience client_secret = None refresh_token = None elif isinstance(base_creds, external_account_authorized_user.Credentials): client_id = self._base.client_id client_secret = self._base.client_secret refresh_token = self._base.refresh_token else: raise TypeError("Invalid Credentials") super(WrappedCredentials, self).__init__(access_token=self._base.token, client_id=client_id, client_secret=client_secret, refresh_token=refresh_token, token_expiry=self._base.expiry, token_uri=None, user_agent=None) def _do_refresh_request(self, http): self._base.refresh(requests.Request()) if self.store is not None: self.store.locked_put(self) @property def access_token(self): return self._base.token @access_token.setter def access_token(self, value): self._base.token = value @property def token_expiry(self): return self._base.expiry @token_expiry.setter def token_expiry(self, value): self._base.expiry = value def to_json(self): """Utility function that creates JSON repr. of a Credentials object. Returns: string, a JSON representation of this instance, suitable to pass to from_json(). """ serialized_data = super().to_json() deserialized_data = json.loads(serialized_data) deserialized_data['_base'] = copy.copy(self._base.info) deserialized_data['access_token'] = self._base.token deserialized_data['token_expiry'] = _parse_expiry(self.token_expiry) return json.dumps(deserialized_data) @classmethod def for_external_account(cls, filename): creds = _get_external_account_credentials_from_file(filename) return cls(creds) @classmethod def for_external_account_authorized_user(cls, filename): creds = _get_external_account_authorized_user_credentials_from_file( filename) return cls(creds) @classmethod def from_json(cls, json_data): """Instantiate a Credentials object from a JSON description of it. The JSON should have been produced by calling .to_json() on the object. Args: data: dict, A deserialized JSON object. Returns: An instance of a Credentials subclass. """ data = json.loads(json_data) # Rebuild the credentials. base = data.get("_base") # Init base cred. base_creds = None if base.get('type') == 'external_account': base_creds = _get_external_account_credentials_from_info(base) elif base.get('type') == 'external_account_authorized_user': base_creds = _get_external_account_authorized_user_credentials_from_info( base) creds = cls(base_creds) # Inject token and expiry. creds.access_token = data.get("access_token") if (data.get('token_expiry') and not isinstance(data['token_expiry'], datetime.datetime)): try: data['token_expiry'] = datetime.datetime.strptime( data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) except ValueError: data['token_expiry'] = None creds.token_expiry = data.get("token_expiry") return creds def _get_external_account_credentials_from_info(info): if info.get( 'subject_token_type') == 'urn:ietf:params:aws:token-type:aws4_request': # Configuration corresponds to an AWS credentials. return aws.Credentials.from_info(info, scopes=DEFAULT_SCOPES) elif (info.get('credential_source') is not None and info.get('credential_source').get('executable') is not None): # Configuration corresponds to pluggable credentials. return pluggable.Credentials.from_info(info, scopes=DEFAULT_SCOPES) else: # Configuration corresponds to an identity pool credentials. return identity_pool.Credentials.from_info(info, scopes=DEFAULT_SCOPES) def _get_external_account_credentials_from_file(filename): with io.open(filename, "r", encoding="utf-8") as json_file: data = json.load(json_file) return _get_external_account_credentials_from_info(data) def _get_external_account_authorized_user_credentials_from_info(info): return external_account_authorized_user.Credentials.from_info(info) def _get_external_account_authorized_user_credentials_from_file(filename): with io.open(filename, "r", encoding="utf-8") as json_file: data = json.load(json_file) return _get_external_account_authorized_user_credentials_from_info(data) def _parse_expiry(expiry): if expiry and isinstance(expiry, datetime.datetime): return expiry.strftime(oauth2client.client.EXPIRY_FORMAT) else: return None	6.83 KB	2025-06-25 08:19

Edit File:

Terminal