Wala shell

Advanced PHP File Manager

Server IP: 127.0.0.1 | PHP Version: 8.3.30 | Domains Hosted: 54

Name	Size	Last Modified
📁 ..	-	2025-06-25 08:26
📄 README This directory contains library code used by gsutil. Users are cautioned not to write programs that call the internal interfaces defined in here; these interfaces were defined only for use by gsutil, and are subject to change without notice. Moreover, Google supports this library only when used by gsutil, not when the library interfaces are called directly by other programs.	378 B	2025-06-25 08:19
📄 __init__.py # -- coding: utf-8 -- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Package marker file.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import os import pkgutil import sys import tempfile import gslib.exception # pylint: disable=g-import-not-at-top from gslib.utils.version_check import check_python_version_support supported, err = check_python_version_support() if not supported: raise gslib.exception.CommandException(err) coverage_outfile = os.getenv('GSUTIL_COVERAGE_OUTPUT_FILE', None) if coverage_outfile: try: import coverage # pylint: disable=g-import-not-at-top coverage_controller = coverage.coverage(data_file=coverage_outfile, data_suffix=True, auto_data=True, source=['gslib'], omit=[ 'gslib/third_party/', 'gslib/tests/', tempfile.gettempdir() + '*', ]) coverage_controller.start() except ImportError: pass # Directory containing the gslib module. GSLIB_DIR = os.path.dirname(os.path.realpath(__file__)) # Path to gsutil executable. This assumes gsutil is the running script. GSUTIL_PATH = os.path.realpath(sys.argv[0]) # The directory that contains the gsutil executable. GSUTIL_DIR = os.path.dirname(GSUTIL_PATH) # Whether or not this was installed via a package manager like pip, deb, rpm, # etc. If installed by just extracting a tarball or zip file, this will be # False. IS_PACKAGE_INSTALL = True # Whether or not this was installed via setup.py develop mode. This creates a # symlink directly to the source directory. IS_EDITABLE_INSTALL = False # Directory where program files like VERSION and CHECKSUM will be. When # installed via tarball, this is the gsutil directory, but the files are moved # to the gslib directory when installed via setup.py. PROGRAM_FILES_DIR = GSLIB_DIR # The gslib directory will be underneath the gsutil directory when installed # from a tarball, but somewhere else on the machine if installed via setup.py. if (not os.path.isfile(os.path.join(PROGRAM_FILES_DIR, 'VERSION')) and os.path.commonprefix((GSUTIL_DIR, GSLIB_DIR)) == GSUTIL_DIR): IS_PACKAGE_INSTALL = False PROGRAM_FILES_DIR = GSUTIL_DIR # If the module was installed from source using editable mode # (i.e. pip install -e) then the files might be one directory up. if not os.path.isfile(os.path.join(PROGRAM_FILES_DIR, 'VERSION')): PROGRAM_FILES_DIR = os.path.normpath(os.path.join(GSLIB_DIR, '..')) IS_EDITABLE_INSTALL = True # Give USER_AGENT a default value for web doc generation. USER_AGENT = '' def _AddVendoredDepsToPythonPath(): """Fix our Python path so that it correctly finds our vendored libraries.""" vendored_path = os.path.join(GSLIB_DIR, 'vendored') # Similar structure to the THIRD_PARTY_LIBS list in gsutil.py: vendored_lib_dirs = [ ('boto', ''), ('oauth2client', ''), ] # Prepend our vendored libraries to be in the front of the Python path so that # they're found before any system installations that might be present. for libdir, subdir in vendored_lib_dirs: sys.path.insert(0, os.path.join(vendored_path, libdir, subdir)) # This is the location of mock_storage_location module. Not every directory # in this path has an __init__.py file, so we couldn't just run # `from boto.tests.integration.s3 import mock_storage_service`. # # We add this to the end, rather than prepending it, so that if other # modules in this directory have the same name as something in our library, # we find our version first. sys.path.append( os.path.join(vendored_path, 'boto', 'tests', 'integration', 's3')) _AddVendoredDepsToPythonPath() def _GetFileContents(filename): """Tries to find the given filename on disk or via pkgutil.get_data. Args: filename: String name of the file. Returns: A tuple containing the absolute path to the requested file and the file's contents as a string (or None if the file doesn't exist). """ fpath = os.path.join(PROGRAM_FILES_DIR, filename) if os.path.isfile(fpath): with open(fpath, 'r') as f: content = f.read() else: content = pkgutil.get_data('gslib', filename) fpath = None if content is not None: if sys.version_info.major > 2 and isinstance(content, bytes): content = content.decode('utf-8') content = content.strip() return (fpath, content) # Get the version file and store it. VERSION_FILE, VERSION = _GetFileContents('VERSION') if not VERSION: raise gslib.exception.CommandException( 'VERSION file not found. Please reinstall gsutil from scratch') __version__ = VERSION # Get the checksum file and store it. CHECKSUM_FILE, CHECKSUM = _GetFileContents('CHECKSUM') if not CHECKSUM: raise gslib.exception.CommandException( 'CHECKSUM file not found. Please reinstall gsutil from scratch') def GetGsutilVersionModifiedTime(): """Returns unix timestamp of when the VERSION file was last modified.""" if not VERSION_FILE: return 0 return int(os.path.getmtime(VERSION_FILE))	6.3 KB	2025-06-25 08:19
📄 __main__.py #!/usr/bin/env python # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Main module for Google Cloud Storage command line tool.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime import errno import getopt import logging import os import re import signal import socket import sys import textwrap import traceback import six from six.moves import configparser from six.moves import range from google.auth import exceptions as google_auth_exceptions import gslib.exception from gslib.exception import CommandException from gslib.exception import ControlCException from gslib.utils.version_check import check_python_version_support from gslib.utils.arg_helper import GetArgumentsAndOptions from gslib.utils.user_agent_helper import GetUserAgent # Load the gsutil version number and append it to boto.UserAgent so the value is # set before anything instantiates boto. This has to run after THIRD_PARTY_DIR # is modified (done in gsutil.py) but before any calls are made that would cause # boto.s3.Connection to be loaded - otherwise the Connection class would end up # with a static reference to the pre-modified version of the UserAgent field, # so boto requests would not include gsutil/version# in the UserAgent string. import boto import gslib from gslib.utils import system_util, text_util # pylint: disable=g-import-not-at-top # This module also imports boto, and will override the UserAgent global variable # if imported above. from gslib import metrics # We parse the options and arguments here so we can pass the results to the user # agent helper. try: opts, args = GetArgumentsAndOptions() except CommandException as e: reason = e.reason if e.informational else 'CommandException: %s' % e.reason err = '%s\n' % reason try: text_util.print_to_fd(err, end='', file=sys.stderr) except UnicodeDecodeError: # Can happen when outputting invalid Unicode filenames. sys.stderr.write(err) if e: metrics.LogFatalError(e) sys.exit(1) # This calculated user agent can be stored for use in StorageV1. gslib.USER_AGENT = GetUserAgent(args, metrics.MetricsCollector.IsDisabled()) boto.UserAgent += gslib.USER_AGENT # pylint: disable=g-bad-import-order import httplib2 import oauth2client from google_reauth import reauth_creds from google_reauth import errors as reauth_errors from gslib import context_config from gslib import wildcard_iterator from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import BadRequestException from gslib.cloud_api import ProjectIdException from gslib.cloud_api import ServiceException from gslib.command_runner import CommandRunner import apitools.base.py.exceptions as apitools_exceptions from gslib.utils import boto_util from gslib.utils import constants from gslib.utils import system_util from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import InitializeSignalHandling from gslib.sig_handling import RegisterSignalHandler CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass'] # We don't use the oauth2 authentication plugin directly; importing it here # ensures that it's loaded and available by default when an operation requiring # authentication is performed. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin except ImportError: pass DEBUG_WARNING = """ *************************** WARNING ************************* * You are running gsutil with debug output enabled. * Be aware that debug output includes authentication credentials. * Make sure to remove the value of the Authorization header for * each HTTP request printed to the console prior to posting to * a public medium such as a forum post or Stack Overflow. *************************** WARNING ************************* """.lstrip() TRACE_WARNING = """ ************************* WARNING ************************* * You are running gsutil with trace output enabled. * Be aware that trace output includes authentication credentials * and may include the contents of any files accessed during the trace. *************************** WARNING ************************* """.lstrip() HTTP_WARNING = """ ************************* WARNING ************************* * You are running gsutil with the "https_validate_certificates" config * variable set to False. This option should always be set to True in * production environments to protect against man-in-the-middle attacks, * and leaking of user data. ************************* WARNING *************************** """.lstrip() debug_level = 0 test_exception_traces = False # pylint: disable=unused-argument def _CleanupSignalHandler(signal_num, cur_stack_frame): """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM. Note that this method is called after main() has been called, so it has access to all the modules imported at the start of main(). Args: signal_num: Unused, but required in the method signature. cur_stack_frame: Unused, but required in the method signature. """ _Cleanup() if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): gslib.command.TeardownMultiprocessingProcesses() def _Cleanup(): for fname in boto_util.GetCleanupFiles(): try: os.unlink(fname) except: # pylint: disable=bare-except pass def _OutputAndExit(message, exception=None): """Outputs message to stderr and exits gsutil with code 1. This function should only be called in single-process, single-threaded mode. Args: message: Message to print to stderr. exception: The exception that caused gsutil to fail. """ if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS or test_exception_traces: stack_trace = traceback.format_exc() err = ('DEBUG: Exception stack trace:\n %s\n%s\n' % (re.sub('\\n', '\n ', stack_trace), message)) else: err = '%s\n' % message try: text_util.print_to_fd(err, end='', file=sys.stderr) except UnicodeDecodeError: # Can happen when outputting invalid Unicode filenames. sys.stderr.write(err) if exception: metrics.LogFatalError(exception) sys.exit(1) def _OutputUsageAndExit(command_runner): command_runner.RunNamedCommand('help') sys.exit(1) class GsutilFormatter(logging.Formatter): """A logging.Formatter that supports logging microseconds (%f).""" def formatTime(self, record, datefmt=None): if datefmt: return datetime.datetime.fromtimestamp(record.created).strftime(datefmt) # Use default implementation if datefmt is not specified. return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt) def _ConfigureRootLogger(level=logging.INFO): """Similar to logging.basicConfig() except it always adds a handler.""" log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' date_format = '%m%d %H:%M:%S.%f' formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) handler = logging.StreamHandler() handler.setFormatter(formatter) root_logger = logging.getLogger() root_logger.addHandler(handler) root_logger.setLevel(level) def main(): InitializeSignalHandling() # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.utils.parallelism_framework_util # pylint: disable=unused-variable from gcs_oauth2_boto_plugin import oauth2_client from apitools.base.py import credentials_lib # pylint: enable=unused-variable if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() else: gslib.command.InitializeThreadingVariables() # This needs to be done after InitializeMultiprocessingVariables(), since # otherwise we can't call CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( gsutil_client_id, gsutil_client_secret) gcs_oauth2_boto_plugin.oauth2_helper.SetLock( gslib.utils.parallelism_framework_util.CreateLock()) credentials_lib.SetCredentialsCacheFileLock( gslib.utils.parallelism_framework_util.CreateLock()) except ImportError: pass global debug_level global test_exception_traces supported, err = check_python_version_support() if not supported: raise CommandException(err) sys.exit(1) boto_util.MonkeyPatchBoto() system_util.MonkeyPatchHttp() # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not boto_util.BOTO_IS_SECURE: raise CommandException('\n'.join( textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug_level = 0 trace_token = None perf_trace_token = None test_exception_traces = False user_project = None # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _CleanupSignalHandler) try: for o, a in opts: if o in ('-d', '--debug'): # Also causes boto to include httplib header output. debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS else: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o == '-u': user_project = a elif o in ('-v', '--version'): version = True elif o in ('-i', '--impersonate-service-account'): constants.IMPERSONATE_SERVICE_ACCOUNT = a elif o == '--perf-trace-token': perf_trace_token = a elif o == '--trace-token': trace_token = a elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True # Avoid printing extra warnings to stderr regarding long retries by # setting the threshold very high. constants.LONG_RETRY_WARN_SEC = 3600 elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) # Now that any Boto option overrides (via `-o` args) have been parsed, # perform initialization that depends on those options. boto_util.configured_certs_file = (boto_util.ConfigureCertsFile()) metrics.LogCommandParams(global_opts=opts) httplib2.debuglevel = debug_level if trace_token: sys.stderr.write(TRACE_WARNING) if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS: sys.stderr.write(DEBUG_WARNING) _ConfigureRootLogger(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] for config_section in ('Boto', 'GSUtil'): try: config_items.extend(boto.config.items(config_section)) except configparser.NoSectionError: pass for i in range(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % boto_util.GetFriendlyConfigFilePaths()) sys.stderr.write('config: %s\n' % str(config_items)) else: # Non-debug log level. root_logger_level = logging.WARNING if quiet else logging.INFO # oauth2client uses INFO and WARNING logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens), so we bump the threshold one level higher where # appropriate. These log levels work for regular- and quiet-level logging. oa2c_logger_level = logging.WARNING oa2c_multiprocess_file_storage_logger_level = logging.ERROR _ConfigureRootLogger(level=root_logger_level) oauth2client.client.logger.setLevel(oa2c_logger_level) oauth2client.contrib.multiprocess_file_storage.logger.setLevel( oa2c_multiprocess_file_storage_logger_level) # pylint: disable=protected-access oauth2client.transport._LOGGER.setLevel(oa2c_logger_level) reauth_creds._LOGGER.setLevel(oa2c_logger_level) # pylint: enable=protected-access # TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3. # Fixes pyu2f v0.1.3 bug. import six # pylint: disable=g-import-not-at-top six.input = six.moves.input if not boto_util.CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] if command_name != 'test': # Don't initialize mTLS authentication because # tests that need it will do this initialization themselves. context_config.create_context_config(logging.getLogger()) _CheckAndWarnForProxyDifferences() # Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at # what argv[] index the command to match starts. We want it to start at the # value for the path to gsutil, so: # $ gsutil <command> # Should be the 1st argument, so '1' # $ python gsutil <command> # Should be the 2nd argument, so '2' # Both are valid; most users invoke gsutil in the first style, but our # integration and prerelease tests invoke it in the second style, as we need # to specify the Python interpreter used to run gsutil. if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'): return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug_level, trace_token=trace_token, parallel_operations=parallel_operations, perf_trace_token=perf_trace_token, user_project=user_project) finally: _Cleanup() def _CheckAndWarnForProxyDifferences(): # If there are both boto config and environment variable config present for # proxies, unset the environment variable and warn if it differs. boto_port = boto.config.getint('Boto', 'proxy_port', 0) if boto.config.get('Boto', 'proxy', None) or boto_port: for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']: if proxy_env_var in os.environ and os.environ[proxy_env_var]: differing_values = [] proxy_info = boto_util.ProxyInfoFromEnvironmentVar(proxy_env_var) if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None): differing_values.append( 'Boto proxy host: "%s" differs from %s proxy host: "%s"' % (boto.config.get('Boto', 'proxy', None), proxy_env_var, proxy_info.proxy_host)) if (proxy_info.proxy_user != boto.config.get('Boto', 'proxy_user', None)): differing_values.append( 'Boto proxy user: "%s" differs from %s proxy user: "%s"' % (boto.config.get('Boto', 'proxy_user', None), proxy_env_var, proxy_info.proxy_user)) if (proxy_info.proxy_pass != boto.config.get('Boto', 'proxy_pass', None)): differing_values.append( 'Boto proxy password differs from %s proxy password' % proxy_env_var) # Only compare ports if at least one is present, since the # boto logic for selecting default ports has not yet executed. if ((proxy_info.proxy_port or boto_port) and proxy_info.proxy_port != boto_port): differing_values.append( 'Boto proxy port: "%s" differs from %s proxy port: "%s"' % (boto_port, proxy_env_var, proxy_info.proxy_port)) if differing_values: sys.stderr.write('\n'.join( textwrap.wrap( 'WARNING: Proxy configuration is present in both the %s ' 'environment variable and boto configuration, but ' 'configuration differs. boto configuration proxy values will ' 'be used. Differences detected:' % proxy_env_var))) sys.stderr.write('\n%s\n' % '\n'.join(differing_values)) # Regardless of whether the proxy configuration values matched, # delete the environment variable so as not to confuse boto. del os.environ[proxy_env_var] def _HandleUnknownFailure(e): # Called if we fall through all known/handled exceptions. raise _OutputAndExit(message='Failure: %s.' % e, exception=e) def _HandleCommandException(e): if e.informational: _OutputAndExit(message=e.reason, exception=e) else: _OutputAndExit(message='CommandException: %s' % e.reason, exception=e) # pylint: disable=unused-argument def _HandleControlC(signal_num, cur_stack_frame): """Called when user hits ^C. This function prints a brief message instead of the normal Python stack trace (unless -D option is used). Args: signal_num: Signal that was caught. cur_stack_frame: Unused. """ if debug_level >= 2: stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) _OutputAndExit('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n' ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace)), exception=ControlCException()) else: _OutputAndExit('Caught CTRL-C (signal %d) - exiting' % signal_num, exception=ControlCException()) def _HandleSigQuit(signal_num, cur_stack_frame): r"""Called when user hits ^\, so we can force breakpoint a running gsutil.""" import pdb # pylint: disable=g-import-not-at-top pdb.set_trace() def _ConstructAccountProblemHelp(reason): """Constructs a help string for an access control error. Args: reason: e.reason string from caught exception. Returns: Contructed help text. """ default_project_id = boto.config.get_value('GSUtil', 'default_project_id') # pylint: disable=line-too-long, g-inconsistent-quotes acct_help = ( "Your request resulted in an AccountProblem (403) error. Usually this " "happens if you attempt to create a bucket without first having " "enabled billing for the project you are using. Please ensure billing is " "enabled for your project by following the instructions at " "`Google Cloud Platform Console<https://support.google.com/cloud/answer/6158867>`. " ) if default_project_id: acct_help += ( "In the project overview, ensure that the Project Number listed for " "your project matches the project ID (%s) from your boto config file. " % default_project_id) acct_help += ( "If the above doesn't resolve your AccountProblem, please send mail to " "buganizer-system+187143@google.com requesting assistance, noting the " "exact command you ran, the fact that you received a 403 AccountProblem " "error, and your project ID. Please do not post your project ID on " "StackOverflow. " "Note: It's possible to use Google Cloud Storage without enabling " "billing if you're only listing or reading objects for which you're " "authorized, or if you're uploading objects to a bucket billed to a " "project that has billing enabled. But if you're attempting to create " "buckets or upload objects to a bucket owned by your own project, you " "must first enable billing for that project.") return acct_help def _CheckAndHandleCredentialException(e, args): # Provide detail to users who have no boto config file (who might previously # have been using gsutil only for accessing publicly readable buckets and # objects). if (not boto_util.HasConfiguredCredentials() and not boto.config.get_value( 'Tests', 'bypass_anonymous_access_warning', False)): # The check above allows tests to assert that we get a particular, # expected failure, rather than always encountering this error message # when there are no configured credentials. This allows tests to # simulate a second user without permissions, without actually requiring # two separate configured users. if system_util.InvokedViaCloudSdk(): message = '\n'.join( textwrap.wrap( 'You are attempting to access protected data with no configured ' 'credentials. Please visit ' 'https://cloud.google.com/console#/project and sign up for an ' 'account, and then run the "gcloud auth login" command to ' 'configure gsutil to use these credentials.')) else: message = '\n'.join( textwrap.wrap( 'You are attempting to access protected data with no configured ' 'credentials. Please visit ' 'https://cloud.google.com/console#/project and sign up for an ' 'account, and then run the "gsutil config" command to configure ' 'gsutil to use these credentials.')) _OutputAndExit(message=message, exception=e) elif (e.reason and (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or 'account for the specified project has been disabled' in e.reason) and ','.join(args).find('gs://') != -1): _OutputAndExit('\n'.join( textwrap.wrap(_ConstructAccountProblemHelp(e.reason))), exception=e) def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None, headers=None, debug_level=0, trace_token=None, parallel_operations=False, perf_trace_token=None, user_project=None): """Runs the command and handles common exceptions.""" # Note that this method is run at the end of main() and thus has access to # all of the modules imported there. # pylint: disable=g-import-not-at-top try: # Catch ^C so we can print a brief message instead of the normal Python # stack trace. Register as a final signal handler because this handler kills # the main gsutil process (so it must run last). RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True) # Catch ^\ so we can force a breakpoint in a running gsutil. if not system_util.IS_WINDOWS: RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit) return command_runner.RunNamedCommand(command_name, args, headers, debug_level, trace_token, parallel_operations, perf_trace_token=perf_trace_token, collect_analytics=True, user_project=user_project) except AttributeError as e: if str(e).find('secret_access_key') != -1: _OutputAndExit( 'Missing credentials for the given URI(s). Does your ' 'boto config file contain all needed credentials?', exception=e) else: _OutputAndExit(message=str(e), exception=e) except CommandException as e: _HandleCommandException(e) except getopt.GetoptError as e: _HandleCommandException(CommandException(e.msg)) except boto.exception.InvalidUriError as e: _OutputAndExit(message='InvalidUriError: %s.' % e.message, exception=e) except gslib.exception.InvalidUrlError as e: _OutputAndExit(message='InvalidUrlError: %s.' % e.message, exception=e) except boto.auth_handler.NotReadyToAuthenticate as e: _OutputAndExit(message='NotReadyToAuthenticate', exception=e) except gslib.exception.ExternalBinaryError as e: _OutputAndExit(message=str(e), exception=e) except OSError as e: # In Python 3, IOError (next except) is an alias for OSError # Sooo... we need the same logic here if (e.errno == errno.EPIPE or (system_util.IS_WINDOWS and e.errno == errno.EINVAL) and not system_util.IsRunningInteractively()): # If we get a pipe error, this just means that the pipe to stdout or # stderr is broken. This can happen if the user pipes gsutil to a command # that doesn't use the entire output stream. Instead of raising an error, # just swallow it up and exit cleanly. sys.exit(0) else: _OutputAndExit(message='OSError: %s.' % e.strerror, exception=e) except IOError as e: if (e.errno == errno.EPIPE or (system_util.IS_WINDOWS and e.errno == errno.EINVAL) and not system_util.IsRunningInteractively()): # If we get a pipe error, this just means that the pipe to stdout or # stderr is broken. This can happen if the user pipes gsutil to a command # that doesn't use the entire output stream. Instead of raising an error, # just swallow it up and exit cleanly. sys.exit(0) else: raise except wildcard_iterator.WildcardException as e: _OutputAndExit(message=e.reason, exception=e) except ProjectIdException as e: _OutputAndExit( 'You are attempting to perform an operation that requires a ' 'project id, with none configured. Please re-run ' 'gsutil config and make sure to follow the instructions for ' 'finding and entering your default project id.', exception=e) except BadRequestException as e: if e.reason == 'MissingSecurityHeader': _CheckAndHandleCredentialException(e, args) _OutputAndExit(message=e, exception=e) except AccessDeniedException as e: _CheckAndHandleCredentialException(e, args) _OutputAndExit(message=e, exception=e) except ArgumentException as e: _OutputAndExit(message=e, exception=e) except ServiceException as e: _OutputAndExit(message=e, exception=e) except (oauth2client.client.HttpAccessTokenRefreshError, google_auth_exceptions.OAuthError, google_auth_exceptions.RefreshError) as e: if system_util.InvokedViaCloudSdk(): _OutputAndExit( 'Your credentials are invalid. ' 'Please run\n$ gcloud auth login', exception=e) else: _OutputAndExit( 'Your credentials are invalid. For more help, see ' '"gsutil help creds", or re-run the gsutil config command (see ' '"gsutil help config").', exception=e) except apitools_exceptions.HttpError as e: # These should usually be retried by the underlying implementation or # wrapped by CloudApi ServiceExceptions, but if we do get them, # print something useful. _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''), e.content or ''), exception=e) except socket.error as e: if e.args[0] == errno.EPIPE: # Retrying with a smaller file (per suggestion below) works because # the library code send loop (in boto/s3/key.py) can get through the # entire file and then request the HTTP response before the socket # gets closed and the response lost. _OutputAndExit( 'Got a "Broken pipe" error. This can happen to clients using Python ' '2.x, when the server sends an error response and then closes the ' 'socket (see http://bugs.python.org/issue5542). If you are trying to ' 'upload a large object you might retry with a small (say 200k) ' 'object, and see if you get a more specific error code.', exception=e) elif e.args[0] == errno.ECONNRESET and ' '.join(args).contains('s3://'): _OutputAndExit('\n'.join( textwrap.wrap( 'Got a "Connection reset by peer" error. One way this can happen is ' 'when copying data to/from an S3 regional bucket. If you are using a ' 'regional S3 bucket you could try re-running this command using the ' 'regional S3 endpoint, for example ' 's3://s3-<region>.amazonaws.com/your-bucket. For details about this ' 'problem see https://github.com/boto/boto/issues/2207')), exception=e) else: _HandleUnknownFailure(e) except oauth2client.client.FlowExchangeError as e: _OutputAndExit('\n%s\n\n' % '\n'.join( textwrap.wrap( 'Failed to retrieve valid credentials (%s). Make sure you selected and ' 'pasted the ENTIRE authorization code (including any numeric prefix ' "e.g. '4/')." % e)), exception=e) except reauth_errors.ReauthSamlLoginRequiredError: if system_util.InvokedViaCloudSdk(): _OutputAndExit('You must re-authenticate with your SAML IdP. ' 'Please run\n$ gcloud auth login') else: _OutputAndExit('You must re-authenticate with your SAML IdP. ' 'Please run\n$ gsutil config') except Exception as e: # pylint: disable=broad-except config_paths = ', '.join(boto_util.GetFriendlyConfigFilePaths()) # Check for two types of errors related to service accounts. These errors # appear to be the same except for their messages, but they are caused by # different problems and both have unhelpful error messages. Moreover, # the error type belongs to PyOpenSSL, which is not necessarily installed. if 'mac verify failure' in str(e): _OutputAndExit( 'Encountered an error while refreshing access token. ' 'If you are using a service account,\nplease verify that the ' 'gs_service_key_file_password field in your config file(s),' '\n%s, is correct.' % config_paths, exception=e) elif 'asn1 encoding routines' in str(e): _OutputAndExit( 'Encountered an error while refreshing access token. ' 'If you are using a service account,\nplease verify that the ' 'gs_service_key_file field in your config file(s),\n%s, is correct.' % config_paths, exception=e) _HandleUnknownFailure(e) def _PerformTabCompletion(command_runner): """Performs gsutil-specific tab completion for the shell.""" # argparse and argcomplete are bundled with the Google Cloud SDK. # When gsutil is invoked from the Google Cloud SDK, both should be available. try: import argcomplete import argparse except ImportError as e: _OutputAndExit('A library required for performing tab completion was' ' not found.\nCause: %s' % e, exception=e) parser = argparse.ArgumentParser(add_help=False) command_runner.ConfigureCommandArgumentParsers(parser) argcomplete.autocomplete(parser, exit_method=sys.exit) return 0 if __name__ == '__main__': sys.exit(main())	33.59 KB	2025-06-25 08:19
📁 addlhelp	-	2025-06-25 08:26
📄 boto_resumable_upload.py # -- coding: utf-8 -- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """Boto translation layer for resumable uploads. See https://cloud.google.com/storage/docs/resumable-uploads-xml for details. Resumable uploads will retry interrupted uploads, resuming at the byte count completed by the last upload attempt. If too many retries happen with no progress (per configurable num_retries param), the upload will be aborted in the current process. Unlike the boto implementation of resumable upload handler, this class does not directly interact with tracker files. Originally Google wrote and contributed this code to the boto project, then copied that code back into gsutil on the release of gsutil 4.0 which supports both boto and non-boto codepaths for resumable uploads. Any bug fixes made to this file should also be integrated to resumable_upload_handler.py in boto, where applicable. TODO: gsutil-beta: Add a similar comment to the boto code. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import random import re import socket import time import six from six.moves import urllib from six.moves import http_client from boto import config from boto import UserAgent from boto.connection import AWSAuthConnection from boto.exception import ResumableTransferDisposition from boto.exception import ResumableUploadException from gslib.exception import InvalidUrlError from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNumRetries from gslib.utils.constants import XML_PROGRESS_CALLBACKS from gslib.utils.constants import UTF8 if six.PY3: long = int class BotoResumableUpload(object): """Upload helper class for resumable uploads via boto.""" BUFFER_SIZE = 8192 RETRYABLE_EXCEPTIONS = (http_client.HTTPException, IOError, socket.error, socket.gaierror) # (start, end) response indicating service has nothing (upload protocol uses # inclusive numbering). SERVICE_HAS_NOTHING = (0, -1) def __init__(self, tracker_callback, logger, resume_url=None, num_retries=None): """Constructor. Instantiate once for each uploaded file. Args: tracker_callback: Callback function that takes a string argument. Used by caller to track this upload across upload interruption. logger: logging.logger instance to use for debug messages. resume_url: If present, attempt to resume the upload at this URL. num_retries: Number of times to retry the upload making no progress. This count resets every time we make progress, so the upload can span many more than this number of retries. """ if resume_url: self._SetUploadUrl(resume_url) else: self.upload_url = None self.num_retries = num_retries self.service_has_bytes = 0 # Byte count at last service check. # Save upload_start_point in instance state so caller can find how # much was transferred by this ResumableUploadHandler (across retries). self.upload_start_point = None self.tracker_callback = tracker_callback self.logger = logger def _SetUploadUrl(self, url): """Saves URL and resets upload state. Called when we start a new resumable upload or get a new tracker URL for the upload. Args: url: URL string for the upload. Raises InvalidUrlError if URL is syntactically invalid. """ parse_result = urllib.parse.urlparse(url) if (parse_result.scheme.lower() not in ['http', 'https'] or not parse_result.netloc): raise InvalidUrlError('Invalid upload URL (%s)' % url) self.upload_url = url self.upload_url_host = (config.get('Credentials', 'gs_host', None) or parse_result.netloc) self.upload_url_path = '%s?%s' % (parse_result.path, parse_result.query) self.service_has_bytes = 0 def _BuildContentRangeHeader(self, range_spec='', length_spec=''): return 'bytes %s/%s' % (range_spec, length_spec) def _QueryServiceState(self, conn, file_length): """Queries service to find out state of given upload. Note that this method really just makes special case use of the fact that the upload service always returns the current start/end state whenever a PUT doesn't complete. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: HTTP response from sending request. Raises: ResumableUploadException if problem querying service. """ # Send an empty PUT so that service replies with this resumable # transfer's state. put_headers = { 'Content-Range': (self._BuildContentRangeHeader('', file_length)), 'Content-Length': '0' } return AWSAuthConnection.make_request(conn, 'PUT', path=self.upload_url_path, auth_path=self.upload_url_path, headers=put_headers, host=self.upload_url_host) def _QueryServicePos(self, conn, file_length): """Queries service to find out what bytes it currently has. Args: conn: HTTPConnection to use for the query. file_length: Total length of the file. Returns: (service_start, service_end), where the values are inclusive. For example, (0, 2) would mean that the service has bytes 0, 1, and* 2. Raises: ResumableUploadException if problem querying service. """ resp = self._QueryServiceState(conn, file_length) if resp.status == 200: # To handle the boundary condition where the service has the complete # file, we return (service_start, file_length-1). That way the # calling code can always simply read up through service_end. (If we # didn't handle this boundary condition here, the caller would have # to check whether service_end == file_length and read one fewer byte # in that case.) return (0, file_length - 1) # Completed upload. if resp.status != 308: # This means the service didn't have any state for the given # upload ID, which can happen (for example) if the caller saved # the upload URL to a file and then tried to restart the transfer # after that upload ID has gone stale. In that case we need to # start a new transfer (and the caller will then save the new # upload URL to the tracker file). raise ResumableUploadException( 'Got non-308 response (%s) from service state query' % resp.status, ResumableTransferDisposition.START_OVER) got_valid_response = False range_spec = resp.getheader('range') if range_spec: # Parse 'bytes=<from>-<to>' range_spec. m = re.search(r'bytes=(\d+)-(\d+)', range_spec) if m: service_start = long(m.group(1)) service_end = long(m.group(2)) got_valid_response = True else: # No Range header, which means the service does not yet have # any bytes. Note that the Range header uses inclusive 'from' # and 'to' values. Since Range 0-0 would mean that the service # has byte 0, omitting the Range header is used to indicate that # the service doesn't have any bytes. return self.SERVICE_HAS_NOTHING if not got_valid_response: raise ResumableUploadException( 'Couldn\'t parse upload service state query response (%s)' % str(resp.getheaders()), ResumableTransferDisposition.START_OVER) if conn.debug >= 1: self.logger.debug('Service has: Range: %d - %d.', service_start, service_end) return (service_start, service_end) def _StartNewResumableUpload(self, key, headers=None): """Starts a new resumable upload. Args: key: Boto Key representing the object to upload. headers: Headers to use in the upload requests. Raises: ResumableUploadException if any errors occur. """ conn = key.bucket.connection if conn.debug >= 1: self.logger.debug('Starting new resumable upload.') self.service_has_bytes = 0 # Start a new resumable upload by sending a POST request with an # empty body and the "X-Goog-Resumable: start" header. Include any # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length # (and raise an exception if they tried to pass one, since it's # a semantic error to specify it at this point, and if we were to # include one now it would cause the service to expect that many # bytes; the POST doesn't include the actual file bytes We set # the Content-Length in the subsequent PUT, based on the uploaded # file size. post_headers = {} for k in headers: if k.lower() == 'content-length': raise ResumableUploadException( 'Attempt to specify Content-Length header (disallowed)', ResumableTransferDisposition.ABORT) post_headers[k] = headers[k] post_headers[conn.provider.resumable_upload_header] = 'start' resp = conn.make_request('POST', key.bucket.name, key.name, post_headers) # Get upload URL from response 'Location' header. body = resp.read() # Check for various status conditions. if resp.status in [429, 500, 503]: # Retry after a delay. raise ResumableUploadException( 'Got status %d from attempt to start resumable upload. ' 'Will wait/retry' % resp.status, ResumableTransferDisposition.WAIT_BEFORE_RETRY) elif resp.status != 200 and resp.status != 201: raise ResumableUploadException( 'Got status %d from attempt to start resumable upload. ' 'Aborting' % resp.status, ResumableTransferDisposition.ABORT) # Else we got 200 or 201 response code, indicating the resumable # upload was created. upload_url = resp.getheader('Location') if not upload_url: raise ResumableUploadException( 'No resumable upload URL found in resumable initiation ' 'POST response (%s)' % body, ResumableTransferDisposition.WAIT_BEFORE_RETRY) self._SetUploadUrl(upload_url) self.tracker_callback(upload_url) def _UploadFileBytes(self, conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, headers): """Attempts to upload file bytes. Makes a single attempt using an existing resumable upload connection. Args: conn: HTTPConnection from the boto Key. http_conn: Separate HTTPConnection for the transfer. fp: File pointer containing bytes to upload. file_length: Total length of the file. total_bytes_uploaded: The total number of bytes uploaded. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. headers: Headers to be used in the upload requests. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. """ buf = fp.read(self.BUFFER_SIZE) if cb: # The cb_count represents the number of full buffers to send between # cb executions. if num_cb > 2: cb_count = file_length / self.BUFFER_SIZE / (num_cb - 2) elif num_cb < 0: cb_count = -1 else: cb_count = 0 i = 0 cb(total_bytes_uploaded, file_length) # Build resumable upload headers for the transfer. Don't send a # Content-Range header if the file is 0 bytes long, because the # resumable upload protocol uses an inclusive end-range (so, sending # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). put_headers = headers.copy() if headers else {} if file_length: if total_bytes_uploaded == file_length: range_header = self._BuildContentRangeHeader('', file_length) else: range_header = self._BuildContentRangeHeader( '%d-%d' % (total_bytes_uploaded, file_length - 1), file_length) put_headers['Content-Range'] = range_header # Set Content-Length to the total bytes we'll send with this PUT. put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) http_request = AWSAuthConnection.build_base_http_request( conn, 'PUT', path=self.upload_url_path, auth_path=None, headers=put_headers, host=self.upload_url_host) http_conn.putrequest('PUT', http_request.path) for k in put_headers: http_conn.putheader(k, put_headers[k]) http_conn.endheaders() # Turn off debug on http connection so upload content isn't included # in debug stream. http_conn.set_debuglevel(0) while buf: # Some code is duplicated here, but separating the PY2 and PY3 paths makes # this easier to remove PY2 blocks when we move to PY3 only. if six.PY2: http_conn.send(buf) total_bytes_uploaded += len(buf) else: if isinstance(buf, bytes): http_conn.send(buf) total_bytes_uploaded += len(buf) else: # Probably a unicode/str object, try encoding. buf_bytes = buf.encode(UTF8) http_conn.send(buf_bytes) total_bytes_uploaded += len(buf_bytes) if cb: i += 1 if i == cb_count or cb_count == -1: cb(total_bytes_uploaded, file_length) i = 0 buf = fp.read(self.BUFFER_SIZE) # Restore http connection debug level. http_conn.set_debuglevel(conn.debug) if cb: cb(total_bytes_uploaded, file_length) if total_bytes_uploaded != file_length: # Abort (and delete the tracker file) so if the user retries # they'll start a new resumable upload rather than potentially # attempting to pick back up later where we left off. raise ResumableUploadException( 'File changed during upload: EOF at %d bytes of %d byte file.' % (total_bytes_uploaded, file_length), ResumableTransferDisposition.ABORT) resp = http_conn.getresponse() if resp.status == 200: # Success. return (resp.getheader('etag'), resp.getheader('x-goog-generation'), resp.getheader('x-goog-metageneration')) # Retry timeout (408) and status 429, 500 and 503 errors after a delay. elif resp.status in [408, 429, 500, 503]: disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY else: # Catch all for any other error codes. disposition = ResumableTransferDisposition.ABORT raise ResumableUploadException( 'Got response code %d while attempting ' 'upload (%s)' % (resp.status, resp.reason), disposition) def _AttemptResumableUpload(self, key, fp, file_length, headers, cb, num_cb): """Attempts a resumable upload. Args: key: Boto key representing object to upload. fp: File pointer containing upload bytes. file_length: Total length of the upload. headers: Headers to be used in upload requests. cb: Progress callback function that takes (progress, total_size). num_cb: Granularity of the callback (maximum number of times the callback will be called during the file transfer). If negative, perform callback with each buffer read. Returns: (etag, generation, metageneration) from service upon success. Raises: ResumableUploadException if any problems occur. """ (service_start, service_end) = self.SERVICE_HAS_NOTHING conn = key.bucket.connection if self.upload_url: # Try to resume existing resumable upload. try: (service_start, service_end) = (self._QueryServicePos(conn, file_length)) self.service_has_bytes = service_start if conn.debug >= 1: self.logger.debug('Resuming transfer.') except ResumableUploadException as e: if conn.debug >= 1: self.logger.debug('Unable to resume transfer (%s).', e.message) self._StartNewResumableUpload(key, headers) else: self._StartNewResumableUpload(key, headers) # upload_start_point allows the code that instantiated the # ResumableUploadHandler to find out the point from which it started # uploading (e.g., so it can correctly compute throughput). if self.upload_start_point is None: self.upload_start_point = service_end total_bytes_uploaded = service_end + 1 # Start reading from the file based upon the number of bytes that the # server has so far. if total_bytes_uploaded < file_length: fp.seek(total_bytes_uploaded) conn = key.bucket.connection # Get a new HTTP connection (vs conn.get_http_connection(), which reuses # pool connections) because httplib requires a new HTTP connection per # transaction. (Without this, calling http_conn.getresponse() would get # "ResponseNotReady".) http_conn = conn.new_http_connection(self.upload_url_host, conn.port, conn.is_secure) http_conn.set_debuglevel(conn.debug) # Make sure to close http_conn at end so if a local file read # failure occurs partway through service will terminate current upload # and can report that progress on next attempt. try: return self._UploadFileBytes(conn, http_conn, fp, file_length, total_bytes_uploaded, cb, num_cb, headers) except (ResumableUploadException, socket.error): resp = self._QueryServiceState(conn, file_length) if resp.status == 400: raise ResumableUploadException( 'Got 400 response from service state query after failed resumable ' 'upload attempt. This can happen for various reasons, including ' 'specifying an invalid request (e.g., an invalid canned ACL) or ' 'if the file size changed between upload attempts', ResumableTransferDisposition.ABORT) else: raise finally: http_conn.close() def HandleResumableUploadException(self, e, debug): if e.disposition == ResumableTransferDisposition.ABORT_CUR_PROCESS: if debug >= 1: self.logger.debug( 'Caught non-retryable ResumableUploadException (%s); ' 'aborting but retaining tracker file', e.message) raise elif e.disposition == ResumableTransferDisposition.ABORT: if debug >= 1: self.logger.debug( 'Caught non-retryable ResumableUploadException (%s); ' 'aborting and removing tracker file', e.message) raise elif e.disposition == ResumableTransferDisposition.START_OVER: raise else: if debug >= 1: self.logger.debug('Caught ResumableUploadException (%s) - will retry', e.message) def TrackProgressLessIterations(self, service_had_bytes_before_attempt, debug=0): """Tracks the number of iterations without progress. Performs randomized exponential backoff. Args: service_had_bytes_before_attempt: Number of bytes the service had prior to this upload attempt. debug: debug level 0..3 """ # At this point we had a re-tryable failure; see if made progress. if self.service_has_bytes > service_had_bytes_before_attempt: self.progress_less_iterations = 0 # If progress, reset counter. else: self.progress_less_iterations += 1 if self.progress_less_iterations > self.num_retries: # Don't retry any longer in the current process. raise ResumableUploadException( 'Too many resumable upload attempts failed without ' 'progress. You might try this upload again later', ResumableTransferDisposition.ABORT_CUR_PROCESS) # Use binary exponential backoff to desynchronize client requests. sleep_time_secs = min(random.random() (2**self.progress_less_iterations), GetMaxRetryDelay()) if debug >= 1: self.logger.debug( 'Got retryable failure (%d progress-less in a row).\n' 'Sleeping %3.1f seconds before re-trying', self.progress_less_iterations, sleep_time_secs) time.sleep(sleep_time_secs) def SendFile(self, key, fp, size, headers, canned_acl=None, cb=None, num_cb=XML_PROGRESS_CALLBACKS): """Upload a file to a key into a bucket on GS, resumable upload protocol. Args: key: `boto.s3.key.Key` or subclass representing the upload destination. fp: File pointer to upload size: Size of the file to upload. headers: The headers to pass along with the PUT request canned_acl: Optional canned ACL to apply to object. cb: Callback function that will be called to report progress on the upload. The callback should accept two integer parameters, the first representing the number of bytes that have been successfully transmitted to GS, and the second representing the total number of bytes that need to be transmitted. num_cb: (optional) If a callback is specified with the cb parameter, this parameter determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. Providing a negative integer will cause your callback to be called with each buffer read. Raises: ResumableUploadException if a problem occurs during the transfer. """ if not headers: headers = {} # If Content-Type header is present and set to None, remove it. # This is gsutil's way of asking boto to refrain from auto-generating # that header. content_type = 'Content-Type' if content_type in headers and headers[content_type] is None: del headers[content_type] if canned_acl: headers[key.provider.acl_header] = canned_acl headers['User-Agent'] = UserAgent file_length = size debug = key.bucket.connection.debug # Use num-retries from constructor if one was provided; else check # for a value specified in the boto config file; else default to 5. if self.num_retries is None: self.num_retries = GetNumRetries() self.progress_less_iterations = 0 while True: # Retry as long as we're making progress. service_had_bytes_before_attempt = self.service_has_bytes try: # Save generation and metageneration in class state so caller # can find these values, for use in preconditions of future # operations on the uploaded object. _, self.generation, self.metageneration = self._AttemptResumableUpload( key, fp, file_length, headers, cb, num_cb) key.generation = self.generation if debug >= 1: self.logger.debug('Resumable upload complete.') return except self.RETRYABLE_EXCEPTIONS as e: if debug >= 1: self.logger.debug('Caught exception (%s)', e.__repr__()) if isinstance(e, IOError) and e.errno == errno.EPIPE: # Broken pipe error causes httplib to immediately # close the socket (http://bugs.python.org/issue5542), # so we need to close the connection before we resume # the upload (which will cause a new connection to be # opened the next time an HTTP request is sent). key.bucket.connection.connection.close() except ResumableUploadException as e: self.HandleResumableUploadException(e, debug) self.TrackProgressLessIterations(service_had_bytes_before_attempt, debug=debug)	24.69 KB	2025-06-25 08:19
📄 boto_translation.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """XML/boto gsutil Cloud API implementation for GCS and Amazon S3.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import binascii import datetime import errno import json import os import pickle import random import re import socket import tempfile import textwrap import threading import time import xml from xml.dom.minidom import parseString as XmlParseString from xml.sax import _exceptions as SaxExceptions import six from six.moves import http_client import boto from boto import config from boto import handler from boto.gs.cors import Cors from boto.gs.lifecycle import LifecycleConfig from boto.s3.cors import CORSConfiguration as S3Cors from boto.s3.deletemarker import DeleteMarker from boto.s3.lifecycle import Lifecycle as S3Lifecycle from boto.s3.prefix import Prefix from boto.s3.tagging import Tags import boto.exception import boto.utils from gslib.boto_resumable_upload import BotoResumableUpload from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import BadRequestException from gslib.cloud_api import CloudApi from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException # Imported for boto AuthHandler purposes. import gslib.devshell_auth_plugin # pylint: disable=unused-import from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.project_id import GOOG_PROJ_ID_HDR from gslib.project_id import PopulateProjectId from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import StorageUrlFromString from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils import parallelism_framework_util from gslib.utils.boto_util import ConfigureNoOpAuthIfNeeded from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNumRetries from gslib.utils.cloud_api_helper import ListToGetFields from gslib.utils.cloud_api_helper import ValidateDstObjectMetadata from gslib.utils.constants import DEFAULT_FILE_BUFFER_SIZE from gslib.utils.constants import REQUEST_REASON_ENV_VAR from gslib.utils.constants import REQUEST_REASON_HEADER_KEY from gslib.utils.constants import S3_DELETE_MARKER_GUID from gslib.utils.constants import UTF8 from gslib.utils.constants import XML_PROGRESS_CALLBACKS from gslib.utils.hashing_helper import Base64EncodeHash from gslib.utils.hashing_helper import Base64ToHexHash from gslib.utils.metadata_util import AddAcceptEncodingGzipIfNeeded from gslib.utils.parallelism_framework_util import multiprocessing_context from gslib.utils.text_util import EncodeStringAsLong from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import AddS3MarkerAclToObjectMetadata from gslib.utils.translation_helper import CorsTranslation from gslib.utils.translation_helper import CreateBucketNotFoundException from gslib.utils.translation_helper import CreateNotFoundExceptionForObjectWrite from gslib.utils.translation_helper import CreateObjectNotFoundException from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE from gslib.utils.translation_helper import HeadersFromObjectMetadata from gslib.utils.translation_helper import LabelTranslation from gslib.utils.translation_helper import LifecycleTranslation from gslib.utils.translation_helper import REMOVE_CORS_CONFIG from gslib.utils.translation_helper import S3MarkerAclFromObjectMetadata from gslib.utils.translation_helper import UnaryDictToXml from gslib.utils.unit_util import TWO_MIB if six.PY3: long = int TRANSLATABLE_BOTO_EXCEPTIONS = (boto.exception.BotoServerError, boto.exception.InvalidUriError, boto.exception.ResumableDownloadException, boto.exception.ResumableUploadException, boto.exception.StorageCreateError, boto.exception.StorageResponseError) # pylint: disable=global-at-module-level global boto_auth_initialized, boto_auth_initialized_lock # If multiprocessing is available, these will be overridden to process-safe # variables in InitializeMultiprocessingVariables. boto_auth_initialized_lock = threading.Lock() boto_auth_initialized = False NON_EXISTENT_OBJECT_REGEX = re.compile(r'.non-\sexistent\sobject', flags=re.DOTALL) # Determines whether an etag is a valid MD5. MD5_REGEX = re.compile(r'^"[a-fA-F0-9]{32}"$') def _AddCustomEndpointToKey(key): """Update Boto Key object with user config's custom endpoint.""" user_setting_to_key_attribute = { 'gs_host': 'host', 'gs_port': 'port', 'gs_host_header': 'host_header', } for user_setting, key_attribute in user_setting_to_key_attribute.items(): user_setting_value = config.get('Credentials', user_setting, None) if user_setting_value is not None: setattr(key.bucket.connection, key_attribute, user_setting_value) def InitializeMultiprocessingVariables(): # pylint: disable=invalid-name """Perform necessary initialization for multiprocessing. See gslib.command.InitializeMultiprocessingVariables for an explanation of why this is necessary. """ # pylint: disable=global-variable-undefined global boto_auth_initialized, boto_auth_initialized_lock boto_auth_initialized_lock = parallelism_framework_util.CreateLock() boto_auth_initialized = multiprocessing_context.Value('i', 0) class DownloadProxyCallbackHandler(object): """Intermediary callback to keep track of the number of bytes downloaded.""" def __init__(self, start_byte, callback): self._start_byte = start_byte self._callback = callback def call(self, bytes_downloaded, total_size): """Saves necessary data and then calls the given Cloud API callback. Args: bytes_downloaded: Number of bytes processed so far. total_size: Total size of the ongoing operation. """ if self._callback: self._callback(self._start_byte + bytes_downloaded, total_size) class BotoTranslation(CloudApi): """Boto-based XML translation implementation of gsutil Cloud API. This class takes gsutil Cloud API objects, translates them to XML service calls, and translates the results back into gsutil Cloud API objects for use by the caller. This class does not support encryption and ignores encryption and decryption parameters. Behavior when encountering encrypted objects is undefined. TODO: Implement support. This class does not support handling a Requester Pays user project for billing, and any given user project will be ignored. TODO: Support user_project. """ def __init__(self, bucket_storage_uri_class, logger, status_queue, provider=None, credentials=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Provider prefix describing cloud storage provider to connect to. 'gs' and 's3' are supported. Function implementations ignore the provider argument and use this one instead. credentials: Unused. debug: Debug level for the API implementation (0..3). http_headers (dict\|None): Arbitrary headers to be included in every request. trace_token: Unused in this subclass. perf_trace_token: Performance trace token to use when making API calls ('gs' provider only). user_project: Unused in this subclass """ super(BotoTranslation, self).__init__(bucket_storage_uri_class, logger, status_queue, provider=provider, debug=debug, http_headers=http_headers, trace_token=trace_token, perf_trace_token=perf_trace_token) _ = credentials # pylint: disable=global-variable-undefined, global-variable-not-assigned global boto_auth_initialized, boto_auth_initialized_lock with boto_auth_initialized_lock: ConfigureNoOpAuthIfNeeded() if isinstance(boto_auth_initialized, bool): boto_auth_initialized = True else: boto_auth_initialized.value = 1 self.api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') def GetServiceAccountId(self): """Service account credentials unused for S3.""" return None def GetBucket(self, bucket_name, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider bucket_uri = self._StorageUriForBucket(bucket_name) headers = self._CreateBaseHeaders() try: return self._BotoBucketToBucket(bucket_uri.get_bucket(validate=True, headers=headers), fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListBuckets(self, project_id=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider get_fields = ListToGetFields(list_fields=fields) headers = self._CreateBaseHeaders() if self.provider == 'gs': headers[GOOG_PROJ_ID_HDR] = PopulateProjectId(project_id) try: provider_uri = boto.storage_uri( '%s://' % self.provider, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) buckets_iter = provider_uri.get_all_buckets(headers=headers) for bucket in buckets_iter: if self.provider == 's3' and bucket.name.lower() != bucket.name: # S3 listings can return buckets with upper-case names, but boto # can't successfully call them. continue yield self._BotoBucketToBucket(bucket, fields=get_fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider bucket_uri = self._StorageUriForBucket(bucket_name) headers = self._CreateBaseHeaders() self._AddPreconditionsToHeaders(preconditions, headers) try: if metadata.acl: boto_acl = AclTranslation.BotoAclFromMessage(metadata.acl) bucket_uri.set_xml_acl(boto_acl.to_xml(), headers=headers) if canned_acl: canned_acls = bucket_uri.canned_acls() if canned_acl not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % canned_acl) bucket_uri.set_acl(canned_acl, bucket_uri.object_name, headers=headers) if canned_def_acl: canned_acls = bucket_uri.canned_acls() if canned_def_acl not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % canned_def_acl) bucket_uri.set_def_acl(canned_def_acl, bucket_uri.object_name, headers=headers) if metadata.cors: if metadata.cors == REMOVE_CORS_CONFIG: metadata.cors = [] boto_cors = CorsTranslation.BotoCorsFromMessage(metadata.cors) bucket_uri.set_cors(boto_cors, False, headers=headers) if metadata.defaultObjectAcl: boto_acl = AclTranslation.BotoAclFromMessage(metadata.defaultObjectAcl) bucket_uri.set_def_xml_acl(boto_acl.to_xml(), headers=headers) if metadata.labels: boto_tags = LabelTranslation.BotoTagsFromMessage(metadata.labels) # TODO: Define tags-related methods on storage_uri objects. The set_tags # method raises an exception if the response differs from the style of # S3, which uses a 204 response code upon success. That method # should be okay with a 200 instead of a 204 for GS responses. In the # meantime, we invoke the underlying bucket's methods directly and # bypass manually raised exceptions from 200 responses. try: bucket_uri.get_bucket().set_tags(boto_tags, headers=headers) except boto.exception.GSResponseError as e: if e.status != 200: raise if metadata.lifecycle: boto_lifecycle = LifecycleTranslation.BotoLifecycleFromMessage( metadata.lifecycle) bucket_uri.configure_lifecycle(boto_lifecycle, False, headers=headers) if metadata.logging: if metadata.logging.logBucket and metadata.logging.logObjectPrefix: bucket_uri.enable_logging(metadata.logging.logBucket, metadata.logging.logObjectPrefix, False, headers=headers) else: # Logging field is present and empty. Disable logging. bucket_uri.disable_logging(False, headers=headers) if metadata.storageClass: bucket_uri.set_storage_class(metadata.storageClass, headers=headers) if metadata.versioning: bucket_uri.configure_versioning(metadata.versioning.enabled, headers=headers) if metadata.website: main_page_suffix = metadata.website.mainPageSuffix error_page = metadata.website.notFoundPage bucket_uri.set_website_config(main_page_suffix, error_page, headers=headers) return self.GetBucket(bucket_name, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider bucket_uri = self._StorageUriForBucket(bucket_name) location = '' if metadata and metadata.location: location = metadata.location.lower() # Pass storage_class param only if this is a GCS bucket. (In S3 the # storage class is specified on the key object.) headers = self._CreateBaseHeaders() if bucket_uri.scheme == 'gs': headers[GOOG_PROJ_ID_HDR] = PopulateProjectId(project_id) storage_class = '' if metadata and metadata.storageClass: storage_class = metadata.storageClass if (metadata and metadata.retentionPolicy and metadata.retentionPolicy.retentionPeriod): headers['x-goog-bucket-retention-period'] = str( metadata.retentionPolicy.retentionPeriod) try: bucket_uri.create_bucket(headers=headers, location=location, storage_class=storage_class) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) else: try: bucket_uri.create_bucket(headers=headers, location=location) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) return self.GetBucket(bucket_name, fields=fields) def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """See CloudApi class for function doc strings.""" _ = provider, preconditions bucket_uri = self._StorageUriForBucket(bucket_name) headers = self._CreateBaseHeaders() try: bucket_uri.delete_bucket(headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: translated_exception = self._TranslateBotoException( e, bucket_name=bucket_name) if (translated_exception and 'BucketNotEmpty' in translated_exception.reason): try: if bucket_uri.get_versioning_config(): if self.provider == 's3': raise NotEmptyException( 'VersionedBucketNotEmpty (%s). Currently, gsutil does not ' 'support listing or removing S3 DeleteMarkers, so you may ' 'need to delete these using another tool to successfully ' 'delete this bucket.' % bucket_name, status=e.status, ) raise NotEmptyException( 'VersionedBucketNotEmpty (%s)' % bucket_name, status=e.status, ) else: raise NotEmptyException( 'BucketNotEmpty (%s)' % bucket_name, status=e.status, ) except TRANSLATABLE_BOTO_EXCEPTIONS as e2: self._TranslateExceptionAndRaise(e2, bucket_name=bucket_name) elif translated_exception and translated_exception.status == 404: raise NotFoundException('Bucket %s does not exist.' % bucket_name) else: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider get_fields = ListToGetFields(list_fields=fields) bucket_uri = self._StorageUriForBucket(bucket_name) headers = self._CreateBaseHeaders() yield_prefixes = fields is None or 'prefixes' in fields yield_objects = fields is None or any( field.startswith('items/') for field in fields) try: objects_iter = bucket_uri.list_bucket(prefix=prefix or '', delimiter=delimiter or '', all_versions=all_versions, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) try: for key in objects_iter: if yield_prefixes and isinstance(key, Prefix): yield CloudApi.CsObjectOrPrefix(key.name, CloudApi.CsObjectOrPrefixType.PREFIX) elif yield_objects: key_to_convert = key # Listed keys are populated with these fields during bucket listing. key_http_fields = set([ 'bucket', 'etag', 'name', 'updated', 'generation', 'metageneration', 'size', ]) # When fields == None, the caller is requesting all possible fields. # If the caller requested any fields that are not populated by bucket # listing, we'll need to make a separate HTTP call for each object to # get its metadata and populate the remaining fields with the result. if not get_fields or (get_fields and not get_fields.issubset(key_http_fields)): generation = None if getattr(key, 'generation', None): generation = key.generation if getattr(key, 'version_id', None): generation = key.version_id key_to_convert = self._GetBotoKey(bucket_name, key.name, generation=generation) return_object = self._BotoKeyToObject(key_to_convert, fields=get_fields) yield CloudApi.CsObjectOrPrefix(return_object, CloudApi.CsObjectOrPrefixType.OBJECT) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider try: return self._BotoKeyToObject(self._GetBotoKey(bucket_name, object_name, generation=generation), fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _CurryDigester(self, digester_object): """Curries a digester object into a form consumable by boto. Key instantiates its own digesters by calling hash_algs[alg]() [note there are no arguments to this function]. So in order to pass in our caught-up digesters during a resumable download, we need to pass the digester object but don't get to look it up based on the algorithm name. Here we use a lambda to make lookup implicit. Args: digester_object: Input object to be returned by the created function. Returns: A function which when called will return the input object. """ return lambda: digester_object def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """See CloudApi class for function doc strings.""" # This implementation will get the object metadata first if we don't pass it # in via serialization_data. headers = self._CreateBaseHeaders() AddAcceptEncodingGzipIfNeeded(headers, compressed_encoding=compressed_encoding) if end_byte is not None: headers['range'] = 'bytes=%s-%s' % (start_byte, end_byte) elif start_byte > 0: headers['range'] = 'bytes=%s-' % (start_byte) elif start_byte < 0: headers['range'] = 'bytes=%s' % (start_byte) # Since in most cases we already made a call to get the object metadata, # here we avoid an extra HTTP call by unpickling the key. This is coupled # with the implementation in _BotoKeyToObject. if serialization_data: serialization_dict = json.loads(serialization_data) key = pickle.loads(binascii.a2b_base64(serialization_dict['url'])) if self.provider == 'gs': _AddCustomEndpointToKey(key) else: key = self._GetBotoKey(bucket_name, object_name, generation=generation) if digesters and self.provider == 'gs': hash_algs = {} for alg in digesters: hash_algs[alg] = self._CurryDigester(digesters[alg]) else: hash_algs = {} total_size = object_size or 0 if serialization_data: total_size = json.loads(serialization_data)['total_size'] if total_size: num_progress_callbacks = max( int(total_size) / TWO_MIB, XML_PROGRESS_CALLBACKS) else: num_progress_callbacks = XML_PROGRESS_CALLBACKS try: if download_strategy is CloudApi.DownloadStrategy.RESUMABLE: self._PerformResumableDownload(download_stream, start_byte, end_byte, key, headers=headers, callback=progress_callback, num_callbacks=num_progress_callbacks, hash_algs=hash_algs) elif download_strategy is CloudApi.DownloadStrategy.ONE_SHOT: self._PerformSimpleDownload( download_stream, key, progress_callback=progress_callback, num_progress_callbacks=num_progress_callbacks, headers=headers, hash_algs=hash_algs) else: raise ArgumentException('Unsupported DownloadStrategy: %s' % download_strategy) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) if self.provider == 's3': if digesters: class HashToDigester(object): """Wrapper class to expose hash digests. boto creates its own digesters in s3's get_file, returning on-the-fly hashes only by way of key.local_hashes. To propagate the digest back to the caller, this stub class implements the digest() function. """ def __init__(self, hash_val): self.hash_val = hash_val def digest(self): # pylint: disable=invalid-name return self.hash_val for alg_name in digesters: if ((download_strategy == CloudApi.DownloadStrategy.RESUMABLE and start_byte != 0) or not ((getattr(key, 'local_hashes', None) and alg_name in key.local_hashes))): # For resumable downloads, boto does not provide a mechanism to # catch up the hash in the case of a partially complete download. # In this case or in the case where no digest was successfully # calculated, set the digester to None, which indicates that we'll # need to manually calculate the hash from the local file once it # is complete. digesters[alg_name] = None else: # Use the on-the-fly hash. digesters[alg_name] = HashToDigester(key.local_hashes[alg_name]) def _PerformSimpleDownload(self, download_stream, key, progress_callback=None, num_progress_callbacks=XML_PROGRESS_CALLBACKS, headers=None, hash_algs=None): try: key.get_contents_to_file(download_stream, cb=progress_callback, num_cb=num_progress_callbacks, headers=headers, hash_algs=hash_algs) except TypeError: # s3 and mocks do not support hash_algs key.get_contents_to_file(download_stream, cb=progress_callback, num_cb=num_progress_callbacks, headers=headers) def _PerformResumableDownload(self, fp, start_byte, end_byte, key, headers=None, callback=None, num_callbacks=XML_PROGRESS_CALLBACKS, hash_algs=None): """Downloads bytes from key to fp, resuming as needed. Args: fp: File pointer into which data should be downloaded. start_byte: Start byte of the download. end_byte: End byte of the download. key: Key object from which data is to be downloaded headers: Headers to send when retrieving the file callback: (optional) a callback function that will be called to report progress on the download. The callback should accept two integer parameters. The first integer represents the number of bytes that have been successfully transmitted from the service. The second represents the total number of bytes that need to be transmitted. num_callbacks: (optional) If a callback is specified with the callback parameter, this determines the granularity of the callback by defining the maximum number of times the callback will be called during the file transfer. hash_algs: Dict of hash algorithms to apply to downloaded bytes. Raises: ResumableDownloadException on error. """ retryable_exceptions = (http_client.HTTPException, IOError, socket.error, socket.gaierror) debug = key.bucket.connection.debug num_retries = GetNumRetries() progress_less_iterations = 0 last_progress_byte = start_byte while True: # Retry as long as we're making progress. try: cb_handler = DownloadProxyCallbackHandler(start_byte, callback) headers = headers.copy() if 'range' in headers: headers.pop('range') headers['Range'] = 'bytes=%d-%d' % (start_byte, end_byte) # Disable AWSAuthConnection-level retry behavior, since that would # cause downloads to restart from scratch. try: key.get_file(fp, headers, cb_handler.call, num_callbacks, override_num_retries=0, hash_algs=hash_algs) except TypeError: key.get_file(fp, headers, cb_handler.call, num_callbacks, override_num_retries=0) fp.flush() # Download succeeded. return except retryable_exceptions as e: # pylint: disable=catching-non-exception if debug >= 1: self.logger.info('Caught exception (%s)', repr(e)) if isinstance(e, IOError) and e.errno == errno.EPIPE: # Broken pipe error causes httplib to immediately # close the socket (http://bugs.python.org/issue5542), # so we need to close and reopen the key before resuming # the download. if self.provider == 's3': key.get_file(fp, headers, cb_handler.call, num_callbacks, override_num_retries=0) else: # self.provider == 'gs' key.get_file(fp, headers, cb_handler.call, num_callbacks, override_num_retries=0, hash_algs=hash_algs) except boto.exception.ResumableDownloadException as e: if (e.disposition == boto.exception.ResumableTransferDisposition.ABORT_CUR_PROCESS): raise ResumableDownloadException(e.message) else: if debug >= 1: self.logger.info( 'Caught boto.exception.ResumableDownloadException (%s) - will ' 'retry', e.message) # At this point we had a re-tryable failure; see if made progress. start_byte = fp.tell() if start_byte > last_progress_byte: last_progress_byte = start_byte progress_less_iterations = 0 else: progress_less_iterations += 1 if progress_less_iterations > num_retries: # Don't retry any longer in the current process. raise ResumableDownloadException( 'Too many resumable download attempts failed without ' 'progress. You might try this download again later') # Close the key, in case a previous download died partway # through and left data in the underlying key HTTP buffer. # Do this within a try/except block in case the connection is # closed (since key.close() attempts to do a final read, in which # case this read attempt would get an IncompleteRead exception, # which we can safely ignore). try: key.close() except http_client.IncompleteRead: pass sleep_time_secs = min(random.random() (2**progress_less_iterations), GetMaxRetryDelay()) if debug >= 1: self.logger.info( 'Got retryable failure (%d progress-less in a row).\nSleeping %d ' 'seconds before re-trying', progress_less_iterations, sleep_time_secs) time.sleep(sleep_time_secs) def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider object_uri = self._StorageUriForObject(bucket_name, object_name, generation=generation) headers = self._CreateBaseHeaders() meta_headers = HeadersFromObjectMetadata(metadata, self.provider) metadata_plus = {} metadata_minus = set() metadata_changed = False for k, v in six.iteritems(meta_headers): metadata_changed = True if v is None: metadata_minus.add(k) else: metadata_plus[k] = v self._AddPreconditionsToHeaders(preconditions, headers) if metadata_changed: try: object_uri.set_metadata(metadata_plus, metadata_minus, False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) if metadata.acl: boto_acl = AclTranslation.BotoAclFromMessage(metadata.acl) try: object_uri.set_xml_acl(boto_acl.to_xml(), key_name=object_name, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) if canned_acl: canned_acls = object_uri.canned_acls() if canned_acl not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % canned_acl) object_uri.set_acl(canned_acl, object_uri.object_name, headers=headers) return self.GetObjectMetadata(bucket_name, object_name, generation=generation, fields=fields) def _PerformSimpleUpload(self, dst_uri, upload_stream, md5=None, canned_acl=None, progress_callback=None, headers=None): dst_uri.set_contents_from_file(upload_stream, md5=md5, policy=canned_acl, cb=progress_callback, headers=headers, rewind=True) def _PerformStreamingUpload(self, dst_uri, upload_stream, canned_acl=None, progress_callback=None, headers=None): if dst_uri.get_provider().supports_chunked_transfer(): dst_uri.set_contents_from_stream(upload_stream, policy=canned_acl, cb=progress_callback, headers=headers) else: # Provider doesn't support chunked transfer, so copy to a temporary # file. (temp_fh, temp_path) = tempfile.mkstemp() try: with open(temp_path, 'wb') as out_fp: stream_bytes = upload_stream.read(DEFAULT_FILE_BUFFER_SIZE) while stream_bytes: out_fp.write(stream_bytes) stream_bytes = upload_stream.read(DEFAULT_FILE_BUFFER_SIZE) with open(temp_path, 'rb') as in_fp: dst_uri.set_contents_from_file(in_fp, policy=canned_acl, headers=headers) finally: os.close(temp_fh) os.unlink(temp_path) def _PerformResumableUpload(self, key, upload_stream, upload_size, tracker_callback, canned_acl=None, serialization_data=None, progress_callback=None, headers=None): resumable_upload = BotoResumableUpload(tracker_callback, self.logger, resume_url=serialization_data) resumable_upload.SendFile(key, upload_stream, upload_size, canned_acl=canned_acl, cb=progress_callback, headers=headers) def _UploadSetup(self, object_metadata, preconditions=None): """Shared upload implementation. Args: object_metadata: Object metadata describing destination object. preconditions: Optional gsutil Cloud API preconditions. Returns: Headers dictionary, StorageUri for upload (based on inputs) """ ValidateDstObjectMetadata(object_metadata) headers = self._CreateBaseHeaders() headers.update(HeadersFromObjectMetadata(object_metadata, self.provider)) if object_metadata.crc32c: if 'x-goog-hash' in headers: headers['x-goog-hash'] += (',crc32c=%s' % object_metadata.crc32c.rstrip('\n')) else: headers['x-goog-hash'] = ('crc32c=%s' % object_metadata.crc32c.rstrip('\n')) if object_metadata.md5Hash: if 'x-goog-hash' in headers: headers['x-goog-hash'] += (',md5=%s' % object_metadata.md5Hash.rstrip('\n')) else: headers['x-goog-hash'] = ('md5=%s' % object_metadata.md5Hash.rstrip('\n')) if 'content-type' in headers and not headers['content-type']: headers['content-type'] = 'application/octet-stream' self._AddPreconditionsToHeaders(preconditions, headers) dst_uri = self._StorageUriForObject(object_metadata.bucket, object_metadata.name) return headers, dst_uri def _HandleSuccessfulUpload(self, dst_uri, object_metadata, fields=None): """Set ACLs on an uploaded object and return its metadata. Args: dst_uri: Generation-specific StorageUri describing the object. object_metadata: Metadata for the object, including an ACL if applicable. fields: If present, return only these Object metadata fields. Returns: gsutil Cloud API Object metadata. Raises: CommandException if the object was overwritten / deleted concurrently. """ try: # The XML API does not support if-generation-match for GET requests. # Therefore, if the object gets overwritten before the ACL and get_key # operations, the best we can do is warn that it happened. self._SetObjectAcl(object_metadata, dst_uri) headers = self._CreateBaseHeaders() return self._BotoKeyToObject(dst_uri.get_key(headers=headers), fields=fields) except boto.exception.InvalidUriError as e: if e.message and NON_EXISTENT_OBJECT_REGEX.match(e.message.encode(UTF8)): raise CommandException('\n'.join( textwrap.wrap( 'Uploaded object (%s) was deleted or overwritten immediately ' 'after it was uploaded. This can happen if you attempt to upload ' 'to the same object multiple times concurrently.' % dst_uri.uri))) else: raise def _SetObjectAcl(self, object_metadata, dst_uri): """Sets the ACL (if present in object_metadata) on an uploaded object.""" headers = self._CreateBaseHeaders() if object_metadata.acl: boto_acl = AclTranslation.BotoAclFromMessage(object_metadata.acl) dst_uri.set_xml_acl(boto_acl.to_xml(), headers=headers) elif self.provider == 's3': s3_acl = S3MarkerAclFromObjectMetadata(object_metadata) if s3_acl: dst_uri.set_xml_acl(s3_acl, headers=headers) def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, size=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" if gzip_encoded: raise NotImplementedError( 'XML API does not support gzip-encoded uploads.') if self.provider == 's3': # Resumable uploads are not supported for s3. return self.UploadObject(upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, fields=fields, size=size) headers, dst_uri = self._UploadSetup(object_metadata, preconditions=preconditions) if not tracker_callback: raise ArgumentException('No tracker callback function set for ' 'resumable upload of %s' % dst_uri) try: self._PerformResumableUpload(dst_uri.new_key(headers=headers), upload_stream, size, tracker_callback, canned_acl=canned_acl, serialization_data=serialization_data, progress_callback=progress_callback, headers=headers) return self._HandleSuccessfulUpload(dst_uri, object_metadata, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, object_metadata.bucket) self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name, not_found_exception=not_found_exception) def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, progress_callback=None, preconditions=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" if gzip_encoded: raise NotImplementedError('XML API does not suport gzip-encoded uploads.') headers, dst_uri = self._UploadSetup(object_metadata, preconditions=preconditions) try: self._PerformStreamingUpload(dst_uri, upload_stream, canned_acl=canned_acl, progress_callback=progress_callback, headers=headers) return self._HandleSuccessfulUpload(dst_uri, object_metadata, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, object_metadata.bucket) self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name, not_found_exception=not_found_exception) def UploadObject(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, size=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" if gzip_encoded: raise NotImplementedError('XML API does not suport gzip-encoded uploads.') headers, dst_uri = self._UploadSetup(object_metadata, preconditions=preconditions) try: md5 = None if object_metadata.md5Hash: # boto expects hex at index 0, base64 at index 1 md5 = [ Base64ToHexHash(object_metadata.md5Hash), object_metadata.md5Hash.strip('\n"\'') ] self._PerformSimpleUpload(dst_uri, upload_stream, md5=md5, canned_acl=canned_acl, progress_callback=progress_callback, headers=headers) return self._HandleSuccessfulUpload(dst_uri, object_metadata, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, object_metadata.bucket) self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name, not_found_exception=not_found_exception) def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """See CloudApi class for function doc strings.""" _ = provider headers = self._CreateBaseHeaders() self._AddPreconditionsToHeaders(preconditions, headers) uri = self._StorageUriForObject(bucket_name, object_name, generation=generation) try: uri.delete_key(validate=False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider if max_bytes_per_call is not None: raise NotImplementedError('XML API does not suport max_bytes_per_call') dst_uri = self._StorageUriForObject(dst_obj_metadata.bucket, dst_obj_metadata.name) # Usually it's okay to treat version_id and generation as # the same, but in this case the underlying boto call determines the # provider based on the presence of one or the other. src_version_id = None if self.provider == 's3': src_version_id = src_generation src_generation = None headers = self._CreateBaseHeaders() headers.update(HeadersFromObjectMetadata(dst_obj_metadata, self.provider)) self._AddPreconditionsToHeaders(preconditions, headers) if canned_acl: headers[dst_uri.get_provider().acl_header] = canned_acl preserve_acl = True if dst_obj_metadata.acl else False if self.provider == 's3': s3_acl = S3MarkerAclFromObjectMetadata(dst_obj_metadata) if s3_acl: preserve_acl = True # If no destination storage class was specified, we want to not specify a # storage class in our copy request, resulting in the bucket's default # storage class being used. Boto uses a default storage class of STANDARD # for copy_key() calls, so we explicitly use None if no storage class was # specified, which will omit the storage class header in our request. dst_storage_class = dst_obj_metadata.storageClass or None try: new_key = dst_uri.copy_key(src_obj_metadata.bucket, src_obj_metadata.name, preserve_acl=preserve_acl, headers=headers, storage_class=dst_storage_class, src_version_id=src_version_id, src_generation=src_generation) return self._BotoKeyToObject(new_key, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, dst_obj_metadata.bucket, src_provider=self.provider, src_bucket_name=src_obj_metadata.bucket, src_object_name=src_obj_metadata.name, src_generation=src_generation) self._TranslateExceptionAndRaise(e, bucket_name=dst_obj_metadata.bucket, object_name=dst_obj_metadata.name, not_found_exception=not_found_exception) def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" _ = provider ValidateDstObjectMetadata(dst_obj_metadata) dst_obj_name = dst_obj_metadata.name dst_obj_metadata.name = None dst_bucket_name = dst_obj_metadata.bucket dst_obj_metadata.bucket = None headers = self._CreateBaseHeaders() headers.update(HeadersFromObjectMetadata(dst_obj_metadata, self.provider)) self._AddPreconditionsToHeaders(preconditions, headers) if not dst_obj_metadata.contentType: dst_obj_metadata.contentType = DEFAULT_CONTENT_TYPE headers['content-type'] = dst_obj_metadata.contentType dst_uri = self._StorageUriForObject(dst_bucket_name, dst_obj_name) src_components = [] for src_obj in src_objs_metadata: src_uri = self._StorageUriForObject(dst_bucket_name, src_obj.name, generation=src_obj.generation) src_components.append(src_uri) try: dst_uri.compose(src_components, headers=headers) return self.GetObjectMetadata(dst_bucket_name, dst_obj_name, fields=fields) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, dst_obj_metadata.bucket, dst_obj_metadata.name) def _AddPreconditionsToHeaders(self, preconditions, headers): """Adds preconditions (if any) to headers.""" if preconditions and self.provider == 'gs': if preconditions.gen_match is not None: headers['x-goog-if-generation-match'] = str(preconditions.gen_match) if preconditions.meta_gen_match is not None: headers['x-goog-if-metageneration-match'] = str( preconditions.meta_gen_match) def _CreateBaseHeaders(self): """Creates base headers used for all API calls in this class.""" base_headers = {} if self.http_headers: base_headers.update(self.http_headers) if self.provider == 'gs': base_headers['x-goog-api-version'] = self.api_version if self.provider == 'gs' and self.perf_trace_token: base_headers['cookie'] = self.perf_trace_token request_reason = os.environ.get(REQUEST_REASON_ENV_VAR) if request_reason: base_headers[REQUEST_REASON_HEADER_KEY] = request_reason return base_headers def _GetMD5FromETag(self, src_etag): """Returns an MD5 from the etag iff the etag is a valid MD5 hash. Args: src_etag: Object etag for which to return the MD5. Returns: MD5 in hex string format, or None. """ if src_etag and MD5_REGEX.search(src_etag): return src_etag.strip('"\'').lower() def _StorageUriForBucket(self, bucket): """Returns a boto storage_uri for the given bucket name. Args: bucket: Bucket name (string). Returns: Boto storage_uri for the bucket. """ return boto.storage_uri( '%s://%s' % (self.provider, bucket), suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug, validate=False) def _StorageUriForObject(self, bucket, object_name, generation=None): """Returns a boto storage_uri for the given object. Args: bucket: Bucket name (string). object_name: Object name (string). generation: Generation or version_id of object. If None, live version of the object is used. Returns: Boto storage_uri for the object. """ uri_string = '%s://%s/%s' % (self.provider, bucket, object_name) if generation: uri_string += '#%s' % generation return boto.storage_uri( uri_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) def _GetBotoKey(self, bucket_name, object_name, generation=None): """Gets the boto key for an object. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation or version of the object to retrieve. Returns: Boto key for the object. """ object_uri = self._StorageUriForObject(bucket_name, object_name, generation=generation) try: headers = self._CreateBaseHeaders() key = object_uri.get_key(headers=headers) if not key: raise CreateObjectNotFoundException('404', self.provider, bucket_name, object_name, generation=generation) return key except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) # pylint: disable=too-many-statements def _BotoBucketToBucket(self, bucket, fields=None): """Constructs an apitools Bucket from a boto bucket. Args: bucket: Boto bucket. fields: If present, construct the apitools Bucket with only this set of metadata fields. Returns: apitools Bucket. """ bucket_uri = self._StorageUriForBucket(bucket.name) cloud_api_bucket = apitools_messages.Bucket(name=bucket.name, id=bucket.name) headers = self._CreateBaseHeaders() if self.provider == 'gs': if not fields or 'storageClass' in fields: if hasattr(bucket, 'get_storage_class'): cloud_api_bucket.storageClass = bucket.get_storage_class() if not fields or 'acl' in fields: try: for acl in AclTranslation.BotoBucketAclToMessage( bucket.get_acl(headers=headers)): cloud_api_bucket.acl.append(acl) except TRANSLATABLE_BOTO_EXCEPTIONS as e: translated_exception = self._TranslateBotoException( e, bucket_name=bucket.name) if (translated_exception and isinstance(translated_exception, AccessDeniedException)): # JSON API doesn't differentiate between a blank ACL list # and an access denied, so this is intentionally left blank. pass else: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'cors' in fields: try: boto_cors = bucket_uri.get_cors(headers=headers) cloud_api_bucket.cors = CorsTranslation.BotoCorsToMessage(boto_cors) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'defaultObjectAcl' in fields: try: for acl in AclTranslation.BotoObjectAclToMessage( bucket.get_def_acl(headers=headers)): cloud_api_bucket.defaultObjectAcl.append(acl) except TRANSLATABLE_BOTO_EXCEPTIONS as e: translated_exception = self._TranslateBotoException( e, bucket_name=bucket.name) if (translated_exception and isinstance(translated_exception, AccessDeniedException)): # JSON API doesn't differentiate between a blank ACL list # and an access denied, so this is intentionally left blank. pass else: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'encryption' in fields: try: keyname = (bucket_uri.get_encryption_config( headers=headers).default_kms_key_name) if keyname: cloud_api_bucket.encryption = ( apitools_messages.Bucket.EncryptionValue()) cloud_api_bucket.encryption.defaultKmsKeyName = keyname except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'lifecycle' in fields: try: boto_lifecycle = bucket_uri.get_lifecycle_config(headers=headers) cloud_api_bucket.lifecycle = ( LifecycleTranslation.BotoLifecycleToMessage(boto_lifecycle)) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'logging' in fields: try: boto_logging = bucket_uri.get_logging_config(headers=headers) if boto_logging and 'Logging' in boto_logging: logging_config = boto_logging['Logging'] log_object_prefix_present = 'LogObjectPrefix' in logging_config log_bucket_present = 'LogBucket' in logging_config if log_object_prefix_present or log_bucket_present: cloud_api_bucket.logging = apitools_messages.Bucket.LoggingValue() if log_object_prefix_present: cloud_api_bucket.logging.logObjectPrefix = ( logging_config['LogObjectPrefix']) if log_bucket_present: cloud_api_bucket.logging.logBucket = logging_config['LogBucket'] except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'website' in fields: try: boto_website = bucket_uri.get_website_config(headers=headers) if boto_website and 'WebsiteConfiguration' in boto_website: website_config = boto_website['WebsiteConfiguration'] main_page_suffix_present = 'MainPageSuffix' in website_config not_found_page_present = 'NotFoundPage' in website_config if main_page_suffix_present or not_found_page_present: cloud_api_bucket.website = apitools_messages.Bucket.WebsiteValue() if main_page_suffix_present: cloud_api_bucket.website.mainPageSuffix = ( website_config['MainPageSuffix']) if not_found_page_present: cloud_api_bucket.website.notFoundPage = ( website_config['NotFoundPage']) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'location' in fields: cloud_api_bucket.location = bucket_uri.get_location(headers=headers) # End gs-specific field checks. if not fields or 'labels' in fields: try: # TODO: Define tags-related methods on storage_uri objects. In the # meantime, we invoke the underlying bucket's methods directly. try: boto_tags = bucket_uri.get_bucket().get_tags() cloud_api_bucket.labels = ( LabelTranslation.BotoTagsToMessage(boto_tags)) except boto.exception.StorageResponseError as e: # If no tagging config exists, the S3 API returns a 404 (the GS API # returns a 200 with an empty TagSet). If the bucket didn't exist, # we would have failed much earlier in this method, so we know that # it's the tagging config that doesn't exist. if not (self.provider == 's3' and e.status == 404): raise except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket.name) if not fields or 'versioning' in fields: versioning = bucket_uri.get_versioning_config(headers=headers) if versioning: if (self.provider == 's3' and 'Versioning' in versioning and versioning['Versioning'] == 'Enabled'): cloud_api_bucket.versioning = ( apitools_messages.Bucket.VersioningValue(enabled=True)) elif self.provider == 'gs': cloud_api_bucket.versioning = ( apitools_messages.Bucket.VersioningValue(enabled=True)) # For S3 long bucket listing we do not support CORS, lifecycle, website, and # logging translation. The individual commands can be used to get # the XML equivalents for S3. return cloud_api_bucket def _BotoKeyToObject(self, key, fields=None): """Constructs an apitools Object from a boto key. Args: key: Boto key to construct Object from. fields: If present, construct the apitools Object with only this set of metadata fields. Returns: apitools Object corresponding to key. """ custom_metadata = None if not fields or 'metadata' in fields or len( [field for field in fields if field.startswith('metadata/')]) >= 1: custom_metadata = self._TranslateBotoKeyCustomMetadata(key) cache_control = None if not fields or 'cacheControl' in fields: cache_control = getattr(key, 'cache_control', None) component_count = None if not fields or 'componentCount' in fields: component_count = getattr(key, 'component_count', None) content_disposition = None if not fields or 'contentDisposition' in fields: content_disposition = getattr(key, 'content_disposition', None) # Other fields like updated and ACL depend on the generation # of the object, so populate that regardless of whether it was requested. generation = self._TranslateBotoKeyGeneration(key) metageneration = None if not fields or 'metageneration' in fields: metageneration = self._TranslateBotoKeyMetageneration(key) time_created = None if not fields or 'timeCreated' in fields: # Translation code to avoid a dependency on dateutil. time_created = self._TranslateBotoKeyTimestamp(key) etag = None if not fields or 'etag' in fields: etag = getattr(key, 'etag', None) if etag: etag = etag.strip('"\'') crc32c = None if not fields or 'crc32c' in fields: if hasattr(key, 'cloud_hashes') and 'crc32c' in key.cloud_hashes: crc32c = base64.b64encode(key.cloud_hashes['crc32c']).rstrip(b'\n') md5_hash = None if not fields or 'md5Hash' in fields: if hasattr(key, 'cloud_hashes') and 'md5' in key.cloud_hashes: md5_hash = base64.b64encode(key.cloud_hashes['md5']).rstrip(b'\n') elif self._GetMD5FromETag(getattr(key, 'etag', None)): md5_hash = Base64EncodeHash(self._GetMD5FromETag(key.etag)) elif self.provider == 's3': # S3 etags are MD5s for non-multi-part objects, but multi-part objects # (which include all objects >= 5 GB) have a custom checksum # implementation that is not currently supported by gsutil. self.logger.warn( 'Non-MD5 etag (%s) present for key %s, data integrity checks are ' 'not possible.', key.etag, key) # Serialize the boto key in the media link if it is requested. This # way we can later access the key without adding an HTTP call. media_link = None if not fields or 'mediaLink' in fields: media_link = binascii.b2a_base64( pickle.dumps(key, pickle.HIGHEST_PROTOCOL)) size = None if not fields or 'size' in fields: size = key.size or 0 storage_class = None if not fields or 'storageClass' in fields: # TODO: Scrub all callers requesting the storageClass field and then # revert this to storage_class; the base storage_class # attribute calls GET on the bucket if the storage class is not already # populated in the key, which can fail if the user does not have # permission on the bucket. storage_class = getattr(key, '_storage_class', None) if six.PY3: if crc32c and isinstance(crc32c, bytes): crc32c = crc32c.decode('ascii') if md5_hash and isinstance(md5_hash, bytes): md5_hash = md5_hash.decode('ascii') cloud_api_object = apitools_messages.Object( bucket=key.bucket.name, name=key.name, size=size, contentEncoding=key.content_encoding, contentLanguage=key.content_language, contentType=key.content_type, cacheControl=cache_control, contentDisposition=content_disposition, etag=etag, crc32c=crc32c, md5Hash=md5_hash, generation=generation, metageneration=metageneration, componentCount=component_count, timeCreated=time_created, metadata=custom_metadata, mediaLink=media_link, storageClass=storage_class) # Remaining functions amend cloud_api_object. self._TranslateDeleteMarker(key, cloud_api_object) if not fields or 'acl' in fields: generation_str = GenerationFromUrlAndString( StorageUrlFromString(self.provider), generation) self._TranslateBotoKeyAcl(key, cloud_api_object, generation=generation_str) return cloud_api_object def _TranslateBotoKeyCustomMetadata(self, key): """Populates an apitools message from custom metadata in the boto key.""" custom_metadata = None if getattr(key, 'metadata', None): custom_metadata = apitools_messages.Object.MetadataValue( additionalProperties=[]) for k, v in six.iteritems(key.metadata): if k.lower() == 'content-language': # Work around content-language being inserted into custom metadata. continue custom_metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty(key=k, value=v)) return custom_metadata def _TranslateBotoKeyGeneration(self, key): """Returns the generation/version_id number from the boto key if present.""" generation = None if self.provider == 'gs': if getattr(key, 'generation', None): generation = long(key.generation) elif self.provider == 's3': if getattr(key, 'version_id', None): generation = EncodeStringAsLong(key.version_id) return generation def _TranslateBotoKeyMetageneration(self, key): """Returns the metageneration number from the boto key if present.""" metageneration = None if self.provider == 'gs': if getattr(key, 'metageneration', None): metageneration = long(key.metageneration) return metageneration def _TranslateBotoKeyTimestamp(self, key): """Parses the timestamp from the boto key into an datetime object. This avoids a dependency on dateutil. Args: key: Boto key to get timestamp from. Returns: datetime object if string is parsed successfully, None otherwise. """ if key.last_modified: if '.' in key.last_modified: key_us_timestamp = key.last_modified.rstrip('Z') + '000Z' else: key_us_timestamp = key.last_modified.rstrip('Z') + '.000000Z' fmt = '%Y-%m-%dT%H:%M:%S.%fZ' try: return datetime.datetime.strptime(key_us_timestamp, fmt) except ValueError: try: # Try alternate format fmt = '%a, %d %b %Y %H:%M:%S %Z' return datetime.datetime.strptime(key.last_modified, fmt) except ValueError: # Could not parse the time; leave updated as None. return None def _TranslateDeleteMarker(self, key, cloud_api_object): """Marks deleted objects with a metadata value (for S3 compatibility).""" if isinstance(key, DeleteMarker): if not cloud_api_object.metadata: cloud_api_object.metadata = apitools_messages.Object.MetadataValue() cloud_api_object.metadata.additionalProperties = [] cloud_api_object.metadata.additionalProperties.append( apitools_messages.Object.MetadataValue.AdditionalProperty( key=S3_DELETE_MARKER_GUID, value=True)) def _TranslateBotoKeyAcl(self, key, cloud_api_object, generation=None): """Updates cloud_api_object with the ACL from the boto key.""" storage_uri_for_key = self._StorageUriForObject(key.bucket.name, key.name, generation=generation) headers = self._CreateBaseHeaders() try: if self.provider == 'gs': key_acl = storage_uri_for_key.get_acl(headers=headers) # key.get_acl() does not support versioning so we need to use # storage_uri to ensure we're getting the versioned ACL. for acl in AclTranslation.BotoObjectAclToMessage(key_acl): cloud_api_object.acl.append(acl) if self.provider == 's3': key_acl = key.get_xml_acl(headers=headers) # ACLs for s3 are different and we use special markers to represent # them in the gsutil Cloud API. AddS3MarkerAclToObjectMetadata(cloud_api_object, key_acl) except boto.exception.GSResponseError as e: if e.status == 403: # Consume access denied exceptions to mimic JSON behavior of simply # returning None if sufficient permission is not present. The caller # needs to handle the case where the ACL is not populated. pass else: raise def _TranslateExceptionAndRaise(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates a Boto exception and raises the translated or original value. Args: e: Any Exception. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ translated_exception = self._TranslateBotoException( e, bucket_name=bucket_name, object_name=object_name, generation=generation, not_found_exception=not_found_exception) if translated_exception: raise translated_exception # pylint: disable=raising-bad-type else: raise def _TranslateBotoException(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates boto exceptions into their gsutil Cloud API equivalents. Args: e: Any exception in TRANSLATABLE_BOTO_EXCEPTIONS. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Returns: ServiceException for translatable exceptions, None otherwise. Because we're using isinstance, check for subtypes first. """ if isinstance(e, boto.exception.StorageResponseError): if e.status == 400: return BadRequestException(e.code, status=e.status, body=e.body) elif e.status == 401 or e.status == 403: return AccessDeniedException(e.code, status=e.status, body=e.body) elif e.status == 404: if not_found_exception: # The exception is pre-constructed prior to translation; the HTTP # status code isn't available at that time. setattr(not_found_exception, 'status', e.status) return not_found_exception elif bucket_name: if object_name: return CreateObjectNotFoundException(e.status, self.provider, bucket_name, object_name, generation=generation) return CreateBucketNotFoundException(e.status, self.provider, bucket_name) return NotFoundException(e.message, status=e.status, body=e.body) elif e.status == 409 and e.code and 'BucketNotEmpty' in e.code: return NotEmptyException('BucketNotEmpty (%s)' % bucket_name, status=e.status, body=e.body) elif e.status == 410: # 410 errors should always cause us to start over - either the UploadID # has expired or there was a server-side problem that requires starting # the upload over from scratch. return ResumableUploadStartOverException(e.message) elif e.status == 412: return PreconditionException(e.code, status=e.status, body=e.body) if isinstance(e, boto.exception.StorageCreateError): return ServiceException('Bucket already exists.', status=e.status, body=e.body) if isinstance(e, boto.exception.BotoServerError): return ServiceException(e.message, status=e.status, body=e.body) if isinstance(e, boto.exception.InvalidUriError): # Work around textwrap when searching for this string. if e.message and NON_EXISTENT_OBJECT_REGEX.match(e.message): return NotFoundException(e.message, status=404) return InvalidUrlError(e.message) if isinstance(e, boto.exception.ResumableUploadException): if e.disposition == boto.exception.ResumableTransferDisposition.ABORT: return ResumableUploadAbortException(e.message) elif (e.disposition == boto.exception.ResumableTransferDisposition.START_OVER): return ResumableUploadStartOverException(e.message) else: return ResumableUploadException(e.message) if isinstance(e, boto.exception.ResumableDownloadException): return ResumableDownloadException(e.message) return None # For function docstrings, see CloudApiDelegator class. def XmlPassThroughGetAcl(self, storage_url, def_obj_acl=False): """See CloudApiDelegator class for function doc strings.""" try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) if def_obj_acl: return uri.get_def_acl() else: return uri.get_acl() except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def XmlPassThroughSetAcl(self, acl_text, storage_url, canned=True, def_obj_acl=False): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) if canned: if def_obj_acl: canned_acls = uri.canned_acls() if acl_text not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % acl_text) uri.set_def_acl(acl_text, uri.object_name, headers=headers) else: canned_acls = uri.canned_acls() if acl_text not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % acl_text) uri.set_acl(acl_text, uri.object_name, headers=headers) else: if def_obj_acl: uri.set_def_xml_acl(acl_text, uri.object_name, headers=headers) else: uri.set_xml_acl(acl_text, uri.object_name, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) # pylint: disable=catching-non-exception def XmlPassThroughSetCors(self, cors_text, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() # Parse XML document and convert into Cors object. if storage_url.scheme == 's3': cors_obj = S3Cors() else: cors_obj = Cors() h = handler.XmlHandler(cors_obj, None) try: xml.sax.parseString(cors_text, h) except SaxExceptions.SAXParseException as e: raise CommandException( 'Requested CORS is invalid: %s at line %s, ' 'column %s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) uri.set_cors(cors_obj, False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def XmlPassThroughGetCors(self, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) try: cors = uri.get_cors(False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) parsed_xml = xml.dom.minidom.parseString(cors.to_xml().encode(UTF8)) # Pretty-print the XML to make it more easily human editable. return parsed_xml.toprettyxml(indent=' ') def XmlPassThroughGetLifecycle(self, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) lifecycle = uri.get_lifecycle_config(False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) parsed_xml = xml.dom.minidom.parseString(lifecycle.to_xml().encode(UTF8)) # Pretty-print the XML to make it more easily human editable. return parsed_xml.toprettyxml(indent=' ') def XmlPassThroughSetLifecycle(self, lifecycle_text, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() # Parse XML document and convert into lifecycle object. if storage_url.scheme == 's3': lifecycle_obj = S3Lifecycle() else: lifecycle_obj = LifecycleConfig() h = handler.XmlHandler(lifecycle_obj, None) try: xml.sax.parseString(lifecycle_text, h) except SaxExceptions.SAXParseException as e: raise CommandException( 'Requested lifecycle config is invalid: %s at line %s, column %s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) uri.configure_lifecycle(lifecycle_obj, False, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def XmlPassThroughGetLogging(self, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) logging_config_xml = UnaryDictToXml( uri.get_logging_config(headers=headers)) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) return XmlParseString(logging_config_xml).toprettyxml() def XmlPassThroughGetTagging(self, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) # TODO: Define tags-related methods on storage_uri objects. In the # meantime, we invoke the underlying bucket's methods directly. tagging_config_xml = uri.get_bucket().get_xml_tags() except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) return XmlParseString(tagging_config_xml).toprettyxml() def XmlPassThroughSetTagging(self, tagging_text, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() tags_obj = Tags() h = handler.XmlHandler(tags_obj, None) try: xml.sax.parseString(tagging_text, h) except SaxExceptions.SAXParseException as e: raise CommandException( 'Requested labels/tagging config is invalid: %s at line %s, column ' '%s' % (e.getMessage(), e.getLineNumber(), e.getColumnNumber())) try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) # TODO: Define tags-related methods on storage_uri objects. In the # meantime, we invoke the underlying bucket's methods directly. uri.get_bucket().set_tags(tags_obj, headers=headers) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def XmlPassThroughGetWebsite(self, storage_url): """See CloudApiDelegator class for function doc strings.""" headers = self._CreateBaseHeaders() try: uri = boto.storage_uri( storage_url.url_string, suppress_consec_slashes=False, bucket_storage_uri_class=self.bucket_storage_uri_class, debug=self.debug) web_config_xml = UnaryDictToXml(uri.get_website_config(headers=headers)) except TRANSLATABLE_BOTO_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) return XmlParseString(web_config_xml).toprettyxml()	85.52 KB	2025-06-25 08:19
📄 bucket_listing_ref.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Classes for cloud/file references yielded by gsutil iterators.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class BucketListingRef(object): """Base class for a reference to one fully expanded iterator result. This allows polymorphic iteration over wildcard-iterated URLs. The reference contains a fully expanded URL string containing no wildcards and referring to exactly one entity (if a wildcard is contained, it is assumed this is part of the raw string and should never be treated as a wildcard). Each reference represents a Bucket, Object, or Prefix. For filesystem URLs, Objects represent files and Prefixes represent directories. The root_object member contains the underlying object as it was retrieved. It is populated by the calling iterator, which may only request certain fields to reduce the number of server requests. For filesystem URLs, root_object is not populated. """ class _BucketListingRefType(object): """Enum class for describing BucketListingRefs.""" BUCKET = 'bucket' # Cloud bucket OBJECT = 'object' # Cloud object or filesystem file PREFIX = 'prefix' # Cloud bucket subdir or filesystem directory @property def url_string(self): return self._url_string @property def type_name(self): return self._ref_type def IsBucket(self): return self._ref_type == self._BucketListingRefType.BUCKET def IsObject(self): return self._ref_type == self._BucketListingRefType.OBJECT def IsPrefix(self): return self._ref_type == self._BucketListingRefType.PREFIX def __str__(self): return self._url_string class BucketListingBucket(BucketListingRef): """BucketListingRef subclass for buckets.""" def __init__(self, storage_url, root_object=None): """Creates a BucketListingRef of type bucket. Args: storage_url: StorageUrl containing a bucket. root_object: Underlying object metadata, if available. """ super(BucketListingBucket, self).__init__() self._ref_type = self._BucketListingRefType.BUCKET self._url_string = storage_url.url_string self.storage_url = storage_url self.root_object = root_object class BucketListingPrefix(BucketListingRef): """BucketListingRef subclass for prefixes.""" def __init__(self, storage_url, root_object=None): """Creates a BucketListingRef of type prefix. Args: storage_url: StorageUrl containing a prefix. root_object: Underlying object metadata, if available. """ super(BucketListingPrefix, self).__init__() self._ref_type = self._BucketListingRefType.PREFIX self._url_string = storage_url.url_string self.storage_url = storage_url self.root_object = root_object class BucketListingObject(BucketListingRef): """BucketListingRef subclass for objects.""" def __init__(self, storage_url, root_object=None): """Creates a BucketListingRef of type object. Args: storage_url: StorageUrl containing an object. root_object: Underlying object metadata, if available. """ super(BucketListingObject, self).__init__() self._ref_type = self._BucketListingRefType.OBJECT self._url_string = storage_url.url_string self.storage_url = storage_url self.root_object = root_object	3.9 KB	2025-06-25 08:19
📄 cloud_api.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gsutil API for interacting with cloud storage providers.""" from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals class CloudApi(object): """Abstract base class for interacting with cloud storage providers. Implementations of the gsutil Cloud API are not guaranteed to be thread-safe. Behavior when calling a gsutil Cloud API instance simultaneously across threads is undefined and doing so will likely cause errors. Therefore, a separate instance of the gsutil Cloud API should be instantiated per-thread. """ def __init__(self, bucket_storage_uri_class, logger, status_queue, provider=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for interacting with the cloud storage provider. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). http_headers (dict\|None): Arbitrary headers to be included in every request. trace_token: Google internal trace token to pass to the API implementation (string). perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.logger = logger self.status_queue = status_queue self.provider = provider self.debug = debug self.http_headers = http_headers self.trace_token = trace_token self.perf_trace_token = perf_trace_token self.user_project = user_project def GetServiceAccountId(self): """Returns the service account email id.""" raise NotImplementedError('GetServiceAccountId must be overridden.') def GetBucket(self, bucket_name, provider=None, fields=None): """Gets Bucket metadata. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields, for example, ['logging', 'defaultObjectAcl'] Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object. """ raise NotImplementedError('GetBucket must be overloaded') def GetBucketIamPolicy(self, bucket_name, provider=None, fields=None): """Returns an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. """ raise NotImplementedError('GetBucketIamPolicy must be overloaded') def SetBucketIamPolicy(self, bucket_name, policy, provider=None): """Sets an IAM policy for the specified Bucket. Args: bucket_name: Name of the bucket. policy: A Policy object describing the IAM policy. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with the cloud storage providers. Returns: Policy object of the bucket. May differ from input Policy. """ raise NotImplementedError('SetBucketIamPolicy must be overloaded') def SignUrl(self, method, duration, path, generation, logger, region, signed_headers, string_to_sign_debug): """Sign a url using service account's system managed private key. Args: method: The HTTP method to be used with the signed URL. duration: timedelta for which the constructed signed URL should be valid. path: String path to the bucket or object for signing, in the form 'bucket' or 'bucket/object'. generation: If not None, specifies a version of an object for signing. logger: logging.Logger for warning and debug output. region: Geographic region in which the requested resource resides. signed_headers: Dict containing the header info like host content-type etc. string_to_sign_debug: If true AND logger is enabled for debug level, print string to sign to debug. Used to differentiate user's signed URL from the probing permissions-check signed URL. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. CommandException for errors because of invalid account used for signing. Returns: The signed url. """ raise NotImplementedError('SignUrl must be overloaded') def ListBuckets(self, project_id=None, provider=None, fields=None): """Lists bucket metadata for the given project. Args: project_id: Project owning the buckets, default from config if None. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/logging', 'items/defaultObjectAcl']. Note that the WildcardIterator class should be used to list buckets instead of calling this function directly. It amends the fields definition from get-like syntax such as ['logging', 'defaultObjectAcl'] so that the caller does not need to prepend 'items/' or specify fields necessary for listing (like nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over Bucket objects. """ raise NotImplementedError('ListBuckets must be overloaded') def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """Updates bucket metadata for the bucket with patch semantics. Args: bucket_name: Name of bucket to update. metadata: Bucket object defining metadata to be updated. canned_acl: Canned ACL to apply to the bucket. canned_def_acl: Canned default object ACL to apply to the bucket. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('PatchBucket must be overloaded') def LockRetentionPolicy(self, bucket_name, metageneration, provider=None): """Locks the Retention Policy on the bucket. Args: bucket_name: Name of bucket to update. metageneration: Bucket metageneration to use as a precondition. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: None """ raise NotImplementedError('LockRetentionPolicy must be overloaded') def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """Creates a new bucket with the specified metadata. Args: bucket_name: Name of the new bucket. project_id: Project owner of the new bucket, default from config if None. metadata: Bucket object defining new bucket metadata. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Bucket metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Bucket object describing new bucket metadata. """ raise NotImplementedError('CreateBucket must be overloaded') def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """Deletes a bucket. Args: bucket_name: Name of the bucket to delete. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteBucket must be overloaded') class CsObjectOrPrefixType(object): """Enum class for describing CsObjectOrPrefix types.""" OBJECT = 'object' # Cloud object PREFIX = 'prefix' # Cloud bucket subdirectory class CsObjectOrPrefix(object): """Container class for ListObjects results.""" def __init__(self, data, datatype): """Stores a ListObjects result. Args: data: Root object, either an apitools Object or a string Prefix. datatype: CsObjectOrPrefixType of data. """ self.data = data self.datatype = datatype def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """Lists objects (with metadata) and prefixes in a bucket. Args: bucket_name: Bucket containing the objects. prefix: Prefix for directory-like behavior. delimiter: Delimiter for directory-like behavior. all_versions: If true, list all object versions. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these metadata fields for the listing, for example: ['items/acl', 'items/updated', 'prefixes']. Note that the WildcardIterator class should be used to list objects instead of calling this function directly. It amends the fields definition from get-like syntax such as ['acl', 'updated'] so that the caller does not need to prepend 'items/' or specify any fields necessary for listing (such as prefixes or nextPageToken). Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Iterator over CsObjectOrPrefix wrapper class. """ raise NotImplementedError('ListObjects must be overloaded') def GetObjectIamPolicy(self, bucket_name, object_name, generation=None, provider=None, fields=None): """Gets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only the IAM policy fields specified. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object IAM policy. """ raise NotImplementedError('GetObjectIamPolicy must be overloaded') def SetObjectIamPolicy(self, bucket_name, object_name, policy, generation=None, provider=None): """Sets IAM policy for specified Object. Args: bucket_name: Bucket containing the object. object_name: Name of the object. policy: IAM Policy object. generation: Generation of the object to which the IAM policy will apply. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Policy object of the object. May differ from input Policy. """ raise NotImplementedError('SetObjectIamPolicy must be overloaded') def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """Gets object metadata. If decryption is supported by the implementing class, this function will read decryption keys from configuration and appropriately retry requests to encrypted objects with the correct key. Args: bucket_name: Bucket containing the object. object_name: Object name. generation: Generation of the object to retrieve. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields, for example, ['acl', 'updated']. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object. """ raise NotImplementedError('GetObjectMetadata must be overloaded') def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None): """Updates object metadata with patch semantics. Args: bucket_name: Bucket containing the object. object_name: Object name for object. metadata: Object object defining metadata to be updated. canned_acl: Canned ACL to be set on the object. generation: Generation (or version) of the object to update. preconditions: Preconditions for the request. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Updated object metadata. """ raise NotImplementedError('PatchObjectMetadata must be overloaded') class DownloadStrategy(object): """Enum class for specifying download strategy.""" ONE_SHOT = 'oneshot' RESUMABLE = 'resumable' # TODO: Change {de,en}cryption_tuple field names, as they don't actually # accept tuples. def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """Gets object data. Args: bucket_name: Bucket containing the object. object_name: Object name. download_stream: Stream to send the object data to. provider: Cloud storage provider to connect to. If not present, class-wide default is used. generation: Generation of the object to retrieve. object_size: Total size of the object being downloaded. compressed_encoding: If true, object is stored with a compressed encoding. download_strategy: Cloud API download strategy to use for download. start_byte: Starting point for download (for resumable downloads and range requests). Can be set to negative to request a range of bytes (python equivalent of [:-3]) end_byte: Ending byte number, inclusive, for download (for range requests). If None, download the rest of the object. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). serialization_data: Implementation-specific JSON string of a dict containing serialization information for the download. digesters: Dict of {string : digester}, where string is a name of a hash algorithm, and digester is a validation digester that supports update(bytes) and digest() using that algorithm. Implementation can set the digester value to None to indicate bytes were not successfully digested on-the-fly. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting an encrypted object. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Content-encoding string if it was detected that the server sent an encoded object during transfer, None otherwise. """ raise NotImplementedError('GetObjectMedia must be overloaded') def UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Optional object size. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObject must be overloaded') def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata. Args: upload_stream: Stream of object data. May not be seekable. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. preconditions: Preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size), but fills in only bytes_transferred. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectStreaming must be overloaded') def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """Uploads object data and metadata using a resumable upload strategy. Args: upload_stream: Seekable stream of object data. object_metadata: Object metadata for new object. Must include bucket and object name. canned_acl: Optional canned ACL to apply to object. Overrides ACL set in object_metadata. size: Total size of the object. preconditions: Preconditions for the request. serialization_data: Dict of {'url' : UploadURL} allowing for uploads to be resumed. tracker_callback: Callback function taking a upload URL string. Guaranteed to be called when the implementation gets an upload URL, allowing the caller to resume the upload across process breaks by saving the upload URL in a tracker file. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the uploaded object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields when the upload is complete. gzip_encoded: Whether to use gzip transport encoding for the upload. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('UploadObjectResumable must be overloaded') # TODO: Change {de,en}cryption_tuple field names, as they don't actually # accept tuples. def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """Copies an object in the cloud. Args: src_obj_metadata: Object metadata for source object. Must include bucket name, object name, and etag. dst_obj_metadata: Object metadata for new object. Must include bucket and object name. src_generation: Generation of the source object to copy. canned_acl: Optional canned ACL to apply to destination object. Overrides ACL set in dst_obj_metadata. preconditions: Destination object preconditions for the request. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). max_bytes_per_call: Integer describing maximum number of bytes to rewrite per service call. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for encrypting the destination object. decryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting the source object. If supplied without encryption_tuple, destination object will be written without customer-supplied encryption. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Object object for newly created destination object. """ raise NotImplementedError('CopyObject must be overloaded') def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """Composes an object in the cloud. Args: src_objs_metadata: List of ComposeRequest.SourceObjectsValueListEntries specifying the objects to compose. dst_obj_metadata: Metadata for the destination object including bucket and object name. preconditions: Destination object preconditions for the request. encryption_tuple: Optional utils.encryption_helper.CryptoKeyWrapper for decrypting source objects and encrypting the destination object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Object metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Composed object metadata. """ raise NotImplementedError('ComposeObject must be overloaded') def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """Deletes an object. Args: bucket_name: Name of the containing bucket. object_name: Name of the object to delete. preconditions: Preconditions for the request. generation: Generation (or version) of the object to delete; if None, deletes the live object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('DeleteObject must be overloaded') def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): """Creates a notification subscription for changes to objects in a bucket. Args: bucket_name: Bucket containing the objects. address: Address to which to send notifications. channel_id: Unique ID string for the channel. token: If present, token string is delivered with each notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. fields: If present, return only these Channel metadata fields. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Channel object describing the notification subscription. """ raise NotImplementedError('WatchBucket must be overloaded') def StopChannel(self, channel_id, resource_id, provider=None): """Stops a notification channel. Args: channel_id: Unique ID string for the channel. resource_id: Version-agnostic ID string for the channel. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('StopChannel must be overloaded') def ListChannels(self, bucket_name, provider=None): """Lists object change notifications for a bucket. Args: bucket_name: Bucket containing the objects provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ raise NotImplementedError('ListChannels must be overloaded') def GetProjectServiceAccount(self, project_number, provider=None): """Get the GCS-owned service account representing this project. Args: project_number: the project in question. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ServiceException for errors interacting with cloud storage providers. Returns: Service account object (with email_address string field) """ raise NotImplementedError('GetProjectServiceAccount must be overloaded') def CreateNotificationConfig(self, bucket_name, pubsub_topic, payload_format, event_types=None, custom_attributes=None, object_name_prefix=None, provider=None): """Creates a new notification with the specified parameters. Args: bucket_name: (Required) Name of the bucket. pubsub_topic: (Required) Cloud Pub/Sub topic to which to publish. payload_format: (Required) payload format, must be 'JSON' or 'NONE'. event_types: (Opt) List of event type filters, e.g. 'OBJECT_FINALIZE'. custom_attributes: (Opt) Dictionary of custom attributes. object_name_prefix: (Opt) Filter on object name. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Notification object describing new notificationConfig """ raise NotImplementedError('CreateNotificationConfig must be overloaded') def DeleteNotificationConfig(self, bucket_name, notification, provider=None): """Deletes a notification. Args: bucket_name: (Required) Name of the bucket. notification: (Required) Integer name of the notification. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None """ raise NotImplementedError('DeleteNotificationConfig must be overloaded') def ListNotificationConfigs(self, bucket_name, provider=None): """Lists notification configs in a bucket. Args: bucket_name: Name of the bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Yields: List of notification objects. """ raise NotImplementedError('ListNotificationConfig must be overloaded') def CreateHmacKey(self, project_id, service_account_email): """Creates a new HMAC key for the specified service account. Args: project_id: Project ID owning the service account for which the key is to be created. service_account_email: Email address of the service account for which to create a key. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns The key metadata and the secret key material. """ raise NotImplementedError('CreateHmacKey must be overloaded') def DeleteHmacKey(self, project_id, access_id): """Deletes an HMAC key. Args: project_id: Project ID owning the requested key. access_id: Name of the HMAC key to be deleted. Raises: NotImplementedError: not implemented TODO(tuckerkirven) """ raise NotImplementedError('DeleteHmacKey must be overloaded') def GetHmacKey(self, project_id, access_id): """Retrieves an HMAC key's metadata. Args: project_id: Project ID owning the service account of the requested key. access_id: Name of the HMAC key for which the metadata is being requested. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: Metadata for the specified key. """ raise NotImplementedError('GetHmacKey must be overloaded') def ListHmacKeys(self, project_id, service_account_email, show_deleted_keys=False): """Lists HMAC keys matching the criteria. Args: project_id: Name of the project from which to list HMAC keys. service_account_email: If present, only keys for the given service account will be returned. show_deleted_keys: If set, show keys in the DELETED state. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Yields: List of HMAC key metadata objects. """ raise NotImplementedError('ListHmacKeys must be overloaded') def UpdateHmacKey(self, project_id, access_id, state, etag): """Updates the state of an HMAC key. Args: project_id: Project ID owning the service account of the updated key. access_id: Name of the HMAC key being updated. state: The state to which the key should be updated. etag: None or a string matching the key's etag to ensure the appropriate version of the key is updated. Raises: NotImplementedError: not implemented TODO(tuckerkirven) Returns: The updated key metadata. """ raise NotImplementedError('UpdateHmacKey must be overloaded') class Preconditions(object): """Preconditions class for specifying preconditions to cloud API requests.""" def __init__(self, gen_match=None, meta_gen_match=None): """Instantiates a Preconditions object. Args: gen_match: Perform request only if generation of target object matches the given integer. Ignored for bucket requests. meta_gen_match: Perform request only if metageneration of target object/bucket matches the given integer. """ self.gen_match = gen_match self.meta_gen_match = meta_gen_match class EncryptionException(Exception): """Exception raised when an encrypted resource cannot be decrypted.""" class ArgumentException(Exception): """Exception raised when arguments to a Cloud API method are invalid. This exception is never raised as a result of a failed call to a cloud storage provider. """ def __init__(self, reason): Exception.__init__(self) self.reason = reason def __repr__(self): return str(self) def __str__(self): return '%s: %s' % (self.__class__.__name__, self.reason) class ProjectIdException(ArgumentException): """Exception raised when a Project ID argument is required but not present.""" class ServiceException(Exception): """Exception raised when a cloud storage provider request fails. This exception is raised only as a result of a failed remote call. """ def __init__(self, reason, status=None, body=None): Exception.__init__(self) self.reason = reason self.status = status self.body = body def __repr__(self): return str(self) def __str__(self): message = '%s:' % self.__class__.__name__ if self.status: message += ' %s' % self.status message += ' %s' % self.reason if self.body: message += '\n%s' % self.body return message class RetryableServiceException(ServiceException): """Exception class for retryable exceptions.""" class ResumableDownloadException(RetryableServiceException): """Exception raised for res. downloads that can be retried later.""" class ResumableUploadException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ same upload ID.""" class ResumableUploadStartOverException(RetryableServiceException): """Exception raised for res. uploads that can be retried w/ new upload ID.""" class ResumableUploadAbortException(ServiceException): """Exception raised for resumable uploads that cannot be retried later.""" class AuthenticationException(ServiceException): """Exception raised for errors during the authentication process.""" class PreconditionException(ServiceException): """Exception raised for precondition failures.""" class NotFoundException(ServiceException): """Exception raised when a resource is not found (404).""" class BucketNotFoundException(NotFoundException): """Exception raised when a bucket resource is not found (404).""" def __init__(self, reason, bucket_name, status=None, body=None): super(BucketNotFoundException, self).__init__(reason, status=status, body=body) self.bucket_name = bucket_name class NotEmptyException(ServiceException): """Exception raised when trying to delete a bucket is not empty.""" class BadRequestException(ServiceException): """Exception raised for malformed requests. Where it is possible to detect invalid arguments prior to sending them to the server, an ArgumentException should be raised instead. """ class AccessDeniedException(ServiceException): """Exception raised when authenticated user has insufficient access rights. This is raised when the authentication process succeeded but the authenticated user does not have access rights to the requested resource. """ class PublishPermissionDeniedException(ServiceException): """Exception raised when bucket does not have publish permission to a topic. This is raised when a custom attempts to set up a notification config to a Cloud Pub/Sub topic, but their GCS bucket does not have permission to publish to the specified topic. """	40.97 KB	2025-06-25 08:19
📄 cloud_api_delegator.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Gsutil API delegator for interacting with cloud storage providers.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto from boto import config from gslib import context_config from gslib.cloud_api import ArgumentException from gslib.cloud_api import CloudApi from gslib.cs_api_map import ApiMapConstants from gslib.cs_api_map import ApiSelector from gslib.exception import CommandException from gslib.utils import boto_util class CloudApiDelegator(CloudApi): """Class that handles delegating requests to gsutil Cloud API implementations. This class is responsible for determining at runtime which gsutil Cloud API implementation should service the request based on the Cloud storage provider, command-level API support, and configuration file override. During initialization it takes as an argument a gsutil_api_map which maps providers to their default and supported gsutil Cloud API implementations (see comments in cs_api_map for details). Instantiation of multiple delegators per-thread is required for multiprocess and/or multithreaded operations. Calling methods on the same delegator in multiple threads is unsafe. """ def __init__(self, bucket_storage_uri_class, gsutil_api_map, logger, status_queue, provider=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for delegating cloud storage requests. This function has different arguments than the gsutil Cloud API __init__ function because of the delegation responsibilties of this class. Args: bucket_storage_uri_class: boto storage_uri class, used by APIs that provide boto translation or mocking. gsutil_api_map: Map of providers and API selector tuples to api classes which can be used to communicate with those providers. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Default provider prefix describing cloud storage provider to connect to. debug: Debug level for the API implementation (0..3). http_headers (dict\|None): Arbitrary headers to be included in every request. trace_token: Apiary trace token to pass to API. perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this project. """ super(CloudApiDelegator, self).__init__(bucket_storage_uri_class, logger, status_queue, provider=provider, debug=debug, http_headers=http_headers, trace_token=trace_token, perf_trace_token=perf_trace_token, user_project=user_project) self.api_map = gsutil_api_map self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper() self.loaded_apis = {} if not self.api_map[ApiMapConstants.API_MAP]: raise ArgumentException('No apiclass supplied for gsutil Cloud API map.') def _GetApi(self, provider): """Returns a valid CloudApi for use by the caller. This function lazy-loads connection and credentials using the API map and credential store provided during class initialization. Args: provider: Provider to load API for. If None, class-wide default is used. Raises: ArgumentException if there is no matching API available in the API map. Returns: Valid API instance that can be used to communicate with the Cloud Storage provider. """ provider = provider or self.provider if not provider: raise ArgumentException('No provider selected for _GetApi') provider = str(provider) if provider not in self.loaded_apis: self.loaded_apis[provider] = {} api_selector = self.GetApiSelector(provider) if api_selector not in self.loaded_apis[provider]: # Need to load the API. self._LoadApi(provider, api_selector) return self.loaded_apis[provider][api_selector] def _LoadApi(self, provider, api_selector): """Loads a CloudApi into the loaded_apis map for this class. Args: provider: Provider to load the API for. api_selector: cs_api_map.ApiSelector defining the API type. """ if provider not in self.api_map[ApiMapConstants.API_MAP]: raise ArgumentException( 'gsutil Cloud API map contains no entry for provider %s.' % provider) if api_selector not in self.api_map[ApiMapConstants.API_MAP][provider]: raise ArgumentException( 'gsutil Cloud API map does not support API %s for provider %s.' % (api_selector, provider)) self.loaded_apis[provider][api_selector] = ( self.api_map[ApiMapConstants.API_MAP][provider][api_selector]( self.bucket_storage_uri_class, self.logger, self.status_queue, provider=provider, debug=self.debug, http_headers=self.http_headers, trace_token=self.trace_token, perf_trace_token=self.perf_trace_token, user_project=self.user_project)) def GetApiSelector(self, provider=None): """Returns a cs_api_map.ApiSelector based on input and configuration. Args: provider: Provider to return the ApiSelector for. If None, class-wide default is used. Returns: cs_api_map.ApiSelector that will be used for calls to the delegator for this provider. """ selected_provider = provider or self.provider if not selected_provider: raise ArgumentException('No provider selected for CloudApi') if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP] or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] not in self.api_map[ApiMapConstants.API_MAP][selected_provider]): raise ArgumentException('No default api available for provider %s' % selected_provider) if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]: raise ArgumentException('No supported apis available for provider %s' % selected_provider) api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] using_gs_hmac = provider == 'gs' and boto_util.UsingGsHmac() configured_encryption = (provider == 'gs' and (config.has_option('GSUtil', 'encryption_key') or config.has_option('GSUtil', 'decryption_key1'))) if using_gs_hmac and configured_encryption: raise CommandException( 'gsutil does not support HMAC credentials with customer-supplied ' 'encryption keys (CSEK) or customer-managed KMS encryption keys ' '(CMEK). Please generate and include non-HMAC credentials ' 'in your .boto configuration file, or to access public encrypted ' 'objects, remove your HMAC credentials.') # If we have only HMAC credentials for Google Cloud Storage, we must use # the XML API as the JSON API does not support HMAC. # # Technically if we have only HMAC credentials, we should still be able to # access public read resources via the JSON API, but the XML API can do # that just as well. It is better to use it than inspect the credentials on # every HTTP call. elif using_gs_hmac: api = ApiSelector.XML # CSEK and CMEK encryption keys are currently only supported in the # JSON API implementation (GcsJsonApi). We can't stop XML API users from # interacting with encrypted objects, since we don't know the object is # encrypted until after the API call is made, but if they specify # configuration values we will use JSON. elif configured_encryption: api = ApiSelector.JSON # Try to force the user's preference to a supported API. elif self.prefer_api in ( self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]): api = self.prefer_api if (api == ApiSelector.XML and context_config.get_context_config() and context_config.get_context_config().use_client_certificate): raise ArgumentException( 'User enabled mTLS by setting "use_client_certificate", but mTLS' ' is not supported for the selected XML API. Try configuring for ' ' the GCS JSON API or setting "use_client_certificate" to "False" in' ' the Boto config.') return api def GetServiceAccountId(self, provider=None): return self._GetApi(provider).GetServiceAccountId() # For function docstrings, see CloudApi class. def GetBucket(self, bucket_name, provider=None, fields=None): return self._GetApi(provider).GetBucket(bucket_name, fields=fields) def GetBucketIamPolicy(self, bucket_name, provider=None, fields=None): return self._GetApi(provider).GetBucketIamPolicy(bucket_name, fields=fields) def SetBucketIamPolicy(self, bucket_name, policy, provider=None): return self._GetApi(provider).SetBucketIamPolicy(bucket_name, policy) def ListBuckets(self, project_id=None, provider=None, fields=None): return self._GetApi(provider).ListBuckets(project_id=project_id, fields=fields) def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): return self._GetApi(provider).PatchBucket(bucket_name, metadata, canned_acl=canned_acl, canned_def_acl=canned_def_acl, preconditions=preconditions, fields=fields) def LockRetentionPolicy(self, bucket_name, metageneration, provider=None): return self._GetApi(provider).LockRetentionPolicy(bucket_name, metageneration, provider=provider) def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): return self._GetApi(provider).CreateBucket(bucket_name, project_id=project_id, metadata=metadata, fields=fields) def DeleteBucket(self, bucket_name, preconditions=None, provider=None): return self._GetApi(provider).DeleteBucket(bucket_name, preconditions=preconditions) def GetObjectIamPolicy(self, bucket_name, object_name, generation=None, provider=None, fields=None): return self._GetApi(provider).GetObjectIamPolicy(bucket_name, object_name, generation, fields=fields) def SetObjectIamPolicy(self, bucket_name, object_name, policy, generation=None, provider=None): return self._GetApi(provider).SetObjectIamPolicy(bucket_name, object_name, policy, generation) def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): return self._GetApi(provider).ListObjects(bucket_name, prefix=prefix, delimiter=delimiter, all_versions=all_versions, fields=fields) def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): return self._GetApi(provider).GetObjectMetadata(bucket_name, object_name, generation=generation, fields=fields) def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None): return self._GetApi(provider).PatchObjectMetadata( bucket_name, object_name, metadata, canned_acl=canned_acl, generation=generation, preconditions=preconditions, fields=fields) def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): return self._GetApi(provider).GetObjectMedia( bucket_name, object_name, download_stream, compressed_encoding=compressed_encoding, download_strategy=download_strategy, start_byte=start_byte, end_byte=end_byte, generation=generation, object_size=object_size, progress_callback=progress_callback, serialization_data=serialization_data, digesters=digesters, decryption_tuple=decryption_tuple) def UploadObject(self, upload_stream, object_metadata, size=None, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): return self._GetApi(provider).UploadObject( upload_stream, object_metadata, size=size, canned_acl=canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, gzip_encoded=gzip_encoded) def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): return self._GetApi(provider).UploadObjectStreaming( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, gzip_encoded=gzip_encoded) def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, size=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): return self._GetApi(provider).UploadObjectResumable( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, size=size, serialization_data=serialization_data, tracker_callback=tracker_callback, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, gzip_encoded=gzip_encoded) def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): return self._GetApi(provider).CopyObject( src_obj_metadata, dst_obj_metadata, src_generation=src_generation, canned_acl=canned_acl, preconditions=preconditions, progress_callback=progress_callback, max_bytes_per_call=max_bytes_per_call, encryption_tuple=encryption_tuple, decryption_tuple=decryption_tuple, fields=fields) def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): return self._GetApi(provider).ComposeObject( src_objs_metadata, dst_obj_metadata, preconditions=preconditions, encryption_tuple=encryption_tuple, fields=fields) def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): return self._GetApi(provider).DeleteObject(bucket_name, object_name, preconditions=preconditions, generation=generation) def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): return self._GetApi(provider).WatchBucket(bucket_name, address, channel_id, token=token, fields=fields) def StopChannel(self, channel_id, resource_id, provider=None): return self._GetApi(provider).StopChannel(channel_id, resource_id) def ListChannels(self, bucket_name, provider=None): return self._GetApi(provider).ListChannels(bucket_name) def GetProjectServiceAccount(self, project_number, provider=None): return self._GetApi(provider).GetProjectServiceAccount(project_number) def CreateNotificationConfig(self, bucket_name, pubsub_topic, payload_format, event_types=None, custom_attributes=None, object_name_prefix=None, provider=None): return self._GetApi(provider).CreateNotificationConfig( bucket_name, pubsub_topic, payload_format, event_types, custom_attributes, object_name_prefix) def DeleteNotificationConfig(self, bucket_name, notification, provider=None): return self._GetApi(provider).DeleteNotificationConfig( bucket_name, notification) def ListNotificationConfigs(self, bucket_name, provider=None): return self._GetApi(provider).ListNotificationConfigs(bucket_name) def ListBucketAccessControls(self, bucket_name, provider=None): return self._GetApi(provider).ListBucketAccessControls(bucket_name) def ListObjectAccessControls(self, bucket_name, object_name, provider=None): return self._GetApi(provider).ListObjectAccessControls( bucket_name, object_name) def CreateHmacKey(self, project_id, service_account_email, provider=None): return self._GetApi(provider).CreateHmacKey(project_id, service_account_email) def DeleteHmacKey(self, project_id, access_id, provider=None): return self._GetApi(provider).DeleteHmacKey(project_id, access_id) def GetHmacKey(self, project_id, access_id, provider=None): return self._GetApi(provider).GetHmacKey(project_id, access_id) def ListHmacKeys(self, project_id, service_account_email, show_deleted_keys=False, provider=None): return self._GetApi(provider).ListHmacKeys(project_id, service_account_email, show_deleted_keys) def SignUrl(self, provider, method, duration, path, generation, logger, region, signed_headers, string_to_sign_debug): return self._GetApi(provider).SignUrl( method=method, duration=duration, path=path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) def UpdateHmacKey(self, project_id, access_id, state, etag, provider=None): return self._GetApi(provider).UpdateHmacKey(project_id, access_id, state, etag) def XmlPassThroughGetAcl(self, storage_url, def_obj_acl=False, provider=None): """XML compatibility function for getting ACLs. Args: storage_url: StorageUrl object. def_obj_acl: If true, get the default object ACL on a bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: ACL XML for the resource specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetAcl(storage_url, def_obj_acl=def_obj_acl) def XmlPassThroughSetAcl(self, acl_text, storage_url, canned=True, def_obj_acl=False, provider=None): """XML compatibility function for setting ACLs. Args: acl_text: XML ACL or canned ACL string. storage_url: StorageUrl object. canned: If true, acl_text is treated as a canned ACL string. def_obj_acl: If true, set the default object ACL on a bucket. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ self._GetApi(provider).XmlPassThroughSetAcl(acl_text, storage_url, canned=canned, def_obj_acl=def_obj_acl) def XmlPassThroughGetCors(self, storage_url, provider=None): """XML compatibility function for getting CORS configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: CORS configuration XML for the bucket specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetCors(storage_url) def XmlPassThroughSetCors(self, cors_text, storage_url, provider=None): """XML compatibility function for setting CORS configuration on a bucket. Args: cors_text: Raw CORS XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ self._GetApi(provider).XmlPassThroughSetCors(cors_text, storage_url) def XmlPassThroughGetLifecycle(self, storage_url, provider=None): """XML compatibility function for getting lifecycle config on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Lifecycle configuration XML for the bucket specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetLifecycle(storage_url) def XmlPassThroughSetLifecycle(self, lifecycle_text, storage_url, provider=None): """XML compatibility function for setting lifecycle config on a bucket. Args: lifecycle_text: Raw lifecycle configuration XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ self._GetApi(provider).XmlPassThroughSetLifecycle(lifecycle_text, storage_url) def XmlPassThroughGetLogging(self, storage_url, provider=None): """XML compatibility function for getting logging configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Logging configuration XML for the bucket specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetLogging(storage_url) def XmlPassThroughSetTagging(self, tags_text, storage_url, provider=None): """XML compatibility function for setting tagging configuration on a bucket. This passthrough provides support for setting a tagging configuration (equivalent to a label configuration) on a cloud bucket. Args: tags_text: Raw tagging configuration XML string. storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: None. """ return self._GetApi(provider).XmlPassThroughSetTagging( tags_text, storage_url) def XmlPassThroughGetTagging(self, storage_url, provider=None): """XML compatibility function for getting tagging configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Tagging configuration XML for the bucket specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetTagging(storage_url) def XmlPassThroughGetWebsite(self, storage_url, provider=None): """XML compatibility function for getting website configuration on a bucket. Args: storage_url: StorageUrl object. provider: Cloud storage provider to connect to. If not present, class-wide default is used. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: Website configuration XML for the bucket specified by storage_url. """ return self._GetApi(provider).XmlPassThroughGetWebsite(storage_url)	29.87 KB	2025-06-25 08:19
📄 command.py # -- coding: utf-8 -- # Copyright 2010 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base class for gsutil commands. In addition to base class code, this file contains helpers that depend on base class state (such as GetAndPrintAcl) In general, functions that depend on class state and that are used by multiple commands belong in this file. Functions that don't depend on class state belong in util.py, and non-shared helpers belong in individual subclasses. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import codecs from collections import namedtuple import copy import getopt import json import logging import os import signal import sys import textwrap import threading import time import traceback import boto from boto.storage_uri import StorageUri import gslib from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import ServiceException from gslib.cloud_api_delegator import CloudApiDelegator from gslib.cs_api_map import ApiSelector from gslib.cs_api_map import GsutilApiMapFactory from gslib.exception import CommandException from gslib.help_provider import HelpProvider from gslib.metrics import CaptureThreadStatException from gslib.metrics import LogPerformanceSummaryParams from gslib.name_expansion import CopyObjectInfo from gslib.name_expansion import CopyObjectsIterator from gslib.name_expansion import NameExpansionIterator from gslib.name_expansion import NameExpansionResult from gslib.name_expansion import SeekAheadNameExpansionIterator from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadThread from gslib.sig_handling import ChildProcessSignalHandler from gslib.sig_handling import GetCaughtSignals from gslib.sig_handling import KillProcess from gslib.sig_handling import MultithreadedMainSignalHandler from gslib.sig_handling import RegisterSignalHandler from gslib.storage_url import HaveFileUrls from gslib.storage_url import HaveProviderUrls from gslib.storage_url import StorageUrlFromString from gslib.storage_url import UrlsAreForSingleProvider from gslib.storage_url import UrlsAreMixOfBucketsAndObjects from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.thread_message import FinalMessage from gslib.thread_message import MetadataMessage from gslib.thread_message import PerformanceSummaryMessage from gslib.thread_message import ProducerThreadMessage from gslib.ui_controller import MainThreadUIQueue from gslib.ui_controller import UIController from gslib.ui_controller import UIThread from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import GetMaxConcurrentCompressedUploads from gslib.utils.constants import NO_MAX from gslib.utils.constants import UTF8 import gslib.utils.parallelism_framework_util from gslib.utils.parallelism_framework_util import AtomicDict from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.parallelism_framework_util import multiprocessing_context from gslib.utils.parallelism_framework_util import ProcessAndThreadSafeInt from gslib.utils.parallelism_framework_util import PutToQueueWithTimeout from gslib.utils.parallelism_framework_util import SEEK_AHEAD_JOIN_TIMEOUT from gslib.utils.parallelism_framework_util import ShouldProhibitMultiprocessing from gslib.utils.parallelism_framework_util import UI_THREAD_JOIN_TIMEOUT from gslib.utils.parallelism_framework_util import ZERO_TASKS_TO_DO_ARGUMENT from gslib.utils.rsync_util import RsyncDiffToApply from gslib.utils.shim_util import GcloudStorageCommandMixin from gslib.utils.system_util import GetTermLines from gslib.utils.system_util import IS_WINDOWS from gslib.utils.translation_helper import AclTranslation from gslib.utils.translation_helper import GetNonMetadataHeaders from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL from gslib.wildcard_iterator import CreateWildcardIterator from six.moves import queue as Queue # pylint: disable=g-import-not-at-top try: from Crypto import Random as CryptoRandom except ImportError: CryptoRandom = None # pylint: enable=g-import-not-at-top OFFER_GSUTIL_M_SUGGESTION_THRESHOLD = 5 OFFER_GSUTIL_M_SUGGESTION_FREQUENCY = 1000 def CreateOrGetGsutilLogger(command_name): """Fetches a logger with the given name that resembles 'print' output. Initial Logger Configuration: The logger abides by gsutil -d/-D/-DD/-q options. If none of those options were specified at invocation, the returned logger will display all messages logged with level INFO or above. Log propagation is disabled. If a logger with the specified name has already been created and configured, it is not reconfigured, e.g.: foo = CreateOrGetGsutilLogger('foo') # Creates and configures Logger "foo". foo.setLevel(logging.DEBUG) # Change level from INFO to DEBUG foo = CreateOrGetGsutilLogger('foo') # Does not reset level to INFO. Args: command_name: (str) Command name to create logger for. Returns: A logging.Logger object. """ log = logging.getLogger(command_name) # There are some scenarios (e.g. unit tests, commands like `mv` that call # other commands) in which we call this function multiple times. To avoid # adding duplicate handlers or overwriting logger attributes set elsewhere, # we only configure the logger if it's one we haven't configured before (i.e. # one that doesn't have a handler set yet). if not log.handlers: log.propagate = False log.setLevel(logging.root.level) log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter('%(message)s')) log.addHandler(log_handler) return log def _DefaultExceptionHandler(cls, e): cls.logger.exception(e) def _UrlArgChecker(command_instance, url): if not command_instance.exclude_symlinks: return True exp_src_url = url.expanded_storage_url if exp_src_url.IsFileUrl() and os.path.islink(exp_src_url.object_name): command_instance.logger.info('Skipping symbolic link %s...', exp_src_url) return False return True def DummyArgChecker(unused_args): return True def SetAclFuncWrapper(cls, name_expansion_result, thread_state=None): return cls.SetAclFunc(name_expansion_result, thread_state=thread_state) def SetAclExceptionHandler(cls, e): """Exception handler that maintains state about post-completion status.""" cls.logger.error(str(e)) cls.everything_set_okay = False # We will keep this list of all thread- or process-safe queues (except the # global status queue) ever created by the main thread so that we can # forcefully kill them upon shutdown. Otherwise, we encounter a Python bug in # which empty queues block forever on join (which is called as part of the # Python exit function cleanup) under the impression that they are non-empty. # However, this also lets us shut down somewhat more cleanly when interrupted. queues = [] def _CryptoRandomAtFork(): if CryptoRandom and getattr(CryptoRandom, 'atfork', None): # Fixes https://github.com/GoogleCloudPlatform/gsutil/issues/390. The # oauth2client module uses Python's Crypto library when pyOpenSSL isn't # present; that module requires calling atfork() in both the parent and # child process after a new process is forked. CryptoRandom.atfork() def _NewMultiprocessingQueue(): new_queue = multiprocessing_context.Queue(MAX_QUEUE_SIZE) queues.append(new_queue) return new_queue def _NewThreadsafeQueue(): new_queue = Queue.Queue(MAX_QUEUE_SIZE) queues.append(new_queue) return new_queue # The maximum size of a process- or thread-safe queue. Imposing this limit # prevents us from needing to hold an arbitrary amount of data in memory. # However, setting this number too high (e.g., >= 32768 on OS X) can cause # problems on some operating systems. MAX_QUEUE_SIZE = 32500 # Related to the max queue size above, once we cross this threshold of # iterated tasks added to the queue, kick off the SeekAheadThread that will # estimate the total work necessary for the command. DEFAULT_TASK_ESTIMATION_THRESHOLD = 30000 def _GetTaskEstimationThreshold(): return boto.config.getint('GSUtil', 'task_estimation_threshold', DEFAULT_TASK_ESTIMATION_THRESHOLD) # That maximum depth of the tree of recursive calls to command.Apply. This is # an arbitrary limit put in place to prevent developers from accidentally # causing problems with infinite recursion, and it can be increased if needed. MAX_RECURSIVE_DEPTH = 5 # Map from deprecated aliases to the current command and subcommands that # provide the same behavior. # TODO: Remove this map and deprecate old commands on 9/9/14. OLD_ALIAS_MAP = { 'chacl': ['acl', 'ch'], 'getacl': ['acl', 'get'], 'setacl': ['acl', 'set'], 'getcors': ['cors', 'get'], 'setcors': ['cors', 'set'], 'chdefacl': ['defacl', 'ch'], 'getdefacl': ['defacl', 'get'], 'setdefacl': ['defacl', 'set'], 'disablelogging': ['logging', 'set', 'off'], 'enablelogging': ['logging', 'set', 'on'], 'getlogging': ['logging', 'get'], 'getversioning': ['versioning', 'get'], 'setversioning': ['versioning', 'set'], 'getwebcfg': ['web', 'get'], 'setwebcfg': ['web', 'set'] } # Declare all of the module level variables - see # InitializeMultiprocessingVariables for an explanation of why this is # necessary. # pylint: disable=global-at-module-level global manager, consumer_pools, task_queues, caller_id_lock, caller_id_counter global total_tasks, call_completed_map, global_return_values_map global need_pool_or_done_cond, caller_id_finished_count, new_pool_needed global current_max_recursive_level, shared_vars_map, shared_vars_list_map global class_map, worker_checking_level_lock, failure_count, thread_stats global glob_status_queue, ui_controller, concurrent_compressed_upload_lock def InitializeMultiprocessingVariables(): """Initializes module-level variables that will be inherited by subprocesses. On Windows, a multiprocessing.Manager object should only be created within an "if __name__ == '__main__':" block. This function must be called, otherwise every command that calls Command.Apply will fail. While multiprocessing variables are initialized at the beginning of gsutil execution, new processes and threads are created only by calls to Command.Apply. When multiple processes and threads are used, the flow of startup/teardown looks like this: 1. __main__: initializes multiprocessing variables, including any necessary Manager processes (here and in gslib.utils.parallelism_framework_util). 2. __main__: Registers signal handlers for terminating signals responsible for cleaning up multiprocessing variables and manager processes upon exit. 3. Command.Apply registers signal handlers for the main process to kill itself after the cleanup handlers registered by __main__ have executed. 4. If worker processes have not been created for the current level of recursive calls, Command.Apply creates those processes. ---- Parallel operations start here, so steps are no longer numbered. ---- - Command.Apply in the main thread starts the ProducerThread. - The Producer thread adds task arguments to the global task queue. - It optionally starts the SeekAheadThread which estimates total work for the Apply call. - Command.Apply in the main thread starts the UIThread, which will consume messages from the global status queue, process them, and display them to the user. - Each worker process creates a thread pool to perform work. - The worker process registers signal handlers to kill itself in response to a terminating signal. - The main thread of the worker process moves items from the global task queue to the process-local task queue. - Worker threads retrieve items from the process-local task queue, perform the work, and post messages to the global status queue. - Worker threads may themselves call Command.Apply. - This creates a new pool of worker subprocesses with the same size as the main pool. This pool is shared amongst all Command.Apply calls at the given recursion depth. - This reuses the global UIThread, global status queue, and global task queue. - This starts a new ProducerThread. - A SeekAheadThread is not started at this level; only one such thread exists at the top level, and it provides estimates for top-level work only. - The ProducerThread runs out of tasks, or the user signals cancellation. - The ProducerThread cancels the SeekAheadThread (if it is running) via an event. - The ProducerThread enqueues special terminating messages on the global task queue and global status queue, signaling the UI Thread to shut down and the main thread to continue operation. - In the termination case, existing processes exit in response to terminating signals from the main process. ---- Parallel operations end here. ---- 5. Further top-level calls to Command.Apply can be made, which will repeat all of the steps made in #4, except that worker processes will be reused. """ # This list of global variables must exactly match the above list of # declarations. # pylint: disable=global-variable-undefined global manager, consumer_pools, task_queues, caller_id_lock, caller_id_counter global total_tasks, call_completed_map, global_return_values_map, thread_stats global need_pool_or_done_cond, caller_id_finished_count, new_pool_needed global current_max_recursive_level, shared_vars_map, shared_vars_list_map global class_map, worker_checking_level_lock, failure_count, glob_status_queue global concurrent_compressed_upload_lock manager = multiprocessing_context.Manager() # List of ConsumerPools - used during shutdown to clean up child processes. consumer_pools = [] # List of all existing task queues - used by all pools to find the queue # that's appropriate for the given recursive_apply_level. task_queues = [] # Used to assign a globally unique caller ID to each Apply call. caller_id_lock = manager.Lock() caller_id_counter = ProcessAndThreadSafeInt(True) # Map from caller_id to total number of tasks to be completed for that ID. total_tasks = AtomicDict(manager=manager) # Map from caller_id to a boolean which is True iff all its tasks are # finished. call_completed_map = AtomicDict(manager=manager) # Used to keep track of the set of return values for each caller ID. global_return_values_map = AtomicDict(manager=manager) # Condition used to notify any waiting threads that a task has finished or # that a call to Apply needs a new set of consumer processes. need_pool_or_done_cond = manager.Condition() # Lock used to prevent multiple worker processes from asking the main thread # to create a new consumer pool for the same level. worker_checking_level_lock = manager.Lock() # Map from caller_id to the current number of completed tasks for that ID. caller_id_finished_count = AtomicDict(manager=manager) # Used as a way for the main thread to distinguish between being woken up # by another call finishing and being woken up by a call that needs a new set # of consumer processes. new_pool_needed = ProcessAndThreadSafeInt(True) current_max_recursive_level = ProcessAndThreadSafeInt(True) # Map from (caller_id, name) to the value of that shared variable. shared_vars_map = AtomicDict(manager=manager) shared_vars_list_map = AtomicDict(manager=manager) # Map from (process id, thread id) to a _ThreadStat object (see WorkerThread). # Used to keep track of thread idle time and execution time. thread_stats = AtomicDict(manager=manager) # Map from caller_id to calling class. class_map = manager.dict() # Number of tasks that resulted in an exception in calls to Apply(). failure_count = ProcessAndThreadSafeInt(True) # Central queue for status reporting across multiple processes and threads. # It's possible that if many processes and threads are executing small file # writes or metadata changes quickly, performance may be bounded by lock # contention on the queue. Initial testing conducted with # 12 processes 5 threads per process showed little difference. If this # becomes a performance bottleneck in the future, consider creating a queue # per-process and having the UI thread poll all of the queues; that approach # would need to address: # - Queue fairness if one queue grows to be disproportionately large # - Reasonable time correlation with events as they occur # # This queue must be torn down after worker processes/threads and the # UI thread have been torn down. Otherwise, these threads may have # undefined behavior when trying to interact with a non-existent queue. glob_status_queue = manager.Queue(MAX_QUEUE_SIZE) # Semaphore lock used to prevent resource exhaustion when running many # compressed uploads in parallel. concurrent_compressed_upload_lock = manager.BoundedSemaphore( GetMaxConcurrentCompressedUploads()) def TeardownMultiprocessingProcesses(): """Should be called by signal handlers prior to shut down.""" # Shut down all processes in consumer pools in preparation for exiting. ShutDownGsutil() # Shut down command and util's multiprocessing.Manager(). # pylint: disable=global-variable-not-assigned,global-variable-undefined global manager # pylint: enable=global-variable-not-assigned,global-variable-undefined manager.shutdown() gslib.utils.parallelism_framework_util.top_level_manager.shutdown() def InitializeThreadingVariables(): """Initializes module-level variables used when running multi-threaded. When multiprocessing is not available (or on Windows where only 1 process is used), thread-safe analogs to the multiprocessing global variables must be initialized. This function is the thread-safe analog to InitializeMultiprocessingVariables. """ # pylint: disable=global-variable-undefined global global_return_values_map, shared_vars_map, failure_count global caller_id_finished_count, shared_vars_list_map, total_tasks global need_pool_or_done_cond, call_completed_map, class_map, thread_stats global task_queues, caller_id_lock, caller_id_counter, glob_status_queue global worker_checking_level_lock, current_max_recursive_level global concurrent_compressed_upload_lock caller_id_counter = ProcessAndThreadSafeInt(False) caller_id_finished_count = AtomicDict() caller_id_lock = threading.Lock() call_completed_map = AtomicDict() class_map = AtomicDict() current_max_recursive_level = ProcessAndThreadSafeInt(False) failure_count = ProcessAndThreadSafeInt(False) glob_status_queue = Queue.Queue(MAX_QUEUE_SIZE) global_return_values_map = AtomicDict() need_pool_or_done_cond = threading.Condition() shared_vars_list_map = AtomicDict() shared_vars_map = AtomicDict() thread_stats = AtomicDict() task_queues = [] total_tasks = AtomicDict() worker_checking_level_lock = threading.Lock() concurrent_compressed_upload_lock = threading.BoundedSemaphore( GetMaxConcurrentCompressedUploads()) # Each subclass of Command must define a property named 'command_spec' that is # an instance of the following class. CommandSpec = namedtuple( 'CommandSpec', [ # Name of command. 'command_name', # Usage synopsis. 'usage_synopsis', # List of command name aliases. 'command_name_aliases', # Min number of args required by this command. 'min_args', # Max number of args required by this command, or NO_MAX. 'max_args', # Getopt-style string specifying acceptable sub args. 'supported_sub_args', # True if file URLs are acceptable for this command. 'file_url_ok', # True if provider-only URLs are acceptable for this command. 'provider_url_ok', # Index in args of first URL arg. 'urls_start_arg', # List of supported APIs 'gs_api_support', # Default API to use for this command 'gs_default_api', # Private arguments (for internal testing) 'supported_private_args', 'argparse_arguments', ]) class Command(HelpProvider, GcloudStorageCommandMixin): """Base class for all gsutil commands.""" # Each subclass must override this with an instance of CommandSpec. command_spec = None _commands_with_subcommands_and_subopts = ('acl', 'defacl', 'iam', 'kms', 'label', 'logging', 'notification', 'retention', 'web') # This keeps track of the recursive depth of the current call to Apply. recursive_apply_level = 0 # If the multiprocessing module isn't available, we'll use this to keep track # of the caller_id. sequential_caller_id = -1 @staticmethod def CreateCommandSpec(command_name, usage_synopsis=None, command_name_aliases=None, min_args=0, max_args=NO_MAX, supported_sub_args='', file_url_ok=False, provider_url_ok=False, urls_start_arg=0, gs_api_support=None, gs_default_api=None, supported_private_args=None, argparse_arguments=None): """Creates an instance of CommandSpec, with defaults.""" return CommandSpec(command_name=command_name, usage_synopsis=usage_synopsis, command_name_aliases=command_name_aliases or [], min_args=min_args, max_args=max_args, supported_sub_args=supported_sub_args, file_url_ok=file_url_ok, provider_url_ok=provider_url_ok, urls_start_arg=urls_start_arg, gs_api_support=gs_api_support or [ApiSelector.XML], gs_default_api=gs_default_api or ApiSelector.XML, supported_private_args=supported_private_args, argparse_arguments=argparse_arguments or []) # Define a convenience property for command name, since it's used many places. def _GetDefaultCommandName(self): return self.command_spec.command_name command_name = property(_GetDefaultCommandName) def _CalculateUrlsStartArg(self): """Calculate the index in args of the first URL arg. Returns: Index of the first URL arg (according to the command spec). """ return self.command_spec.urls_start_arg def _TranslateDeprecatedAliases(self, args): """Map deprecated aliases to the corresponding new command, and warn.""" new_command_args = OLD_ALIAS_MAP.get(self.command_alias_used, None) if new_command_args: # Prepend any subcommands for the new command. The command name itself # is not part of the args, so leave it out. args = new_command_args[1:] + args self.logger.warn('\n'.join( textwrap.wrap( ('You are using a deprecated alias, "%(used_alias)s", for the ' '"%(command_name)s" command. This will stop working on 9/9/2014. ' 'Please use "%(command_name)s" with the appropriate sub-command in ' 'the future. See "gsutil help %(command_name)s" for details.') % { 'used_alias': self.command_alias_used, 'command_name': self.command_name }))) return args def __init__(self, command_runner, args, headers, debug, trace_token, parallel_operations, bucket_storage_uri_class, gsutil_api_class_map_factory, logging_filters=None, command_alias_used=None, perf_trace_token=None, user_project=None): """Instantiates a Command. Args: command_runner: CommandRunner (for commands built atop other commands). args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the API implementation. parallel_operations: Should command operations be executed in parallel? bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. logging_filters: Optional list of logging. Filters to apply to this command's logger. command_alias_used: The alias that was actually used when running this command (as opposed to the "official" command name, which will always correspond to the file name). perf_trace_token: Performance measurement trace token to use when making API calls. user_project: Project to be billed for this request. Implementation note: subclasses shouldn't need to define an __init__ method, and instead depend on the shared initialization that happens here. If you do define an __init__ method in a subclass you'll need to explicitly call super().__init__(). But you're encouraged not to do this, because it will make changing the __init__ interface more painful. """ # Save class values from constructor params. super().__init__() self.command_runner = command_runner self.unparsed_args = args self.headers = headers self.debug = debug self.trace_token = trace_token self.perf_trace_token = perf_trace_token self.parallel_operations = parallel_operations self.user_project = user_project self.bucket_storage_uri_class = bucket_storage_uri_class self.gsutil_api_class_map_factory = gsutil_api_class_map_factory self.exclude_symlinks = False self.recursion_requested = False self.all_versions = False self.command_alias_used = command_alias_used self.seek_ahead_gsutil_api = None # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global ui_controller # pylint: enable=global-variable-undefined # pylint: enable=global-variable-not-assigned # Global instance of a threaded logger object. self.logger = CreateOrGetGsutilLogger(self.command_name) if logging_filters: for log_filter in logging_filters: self.logger.addFilter(log_filter) if self.headers is not None: self.non_metadata_headers = GetNonMetadataHeaders(self.headers) else: self.non_metadata_headers = None if self.command_spec is None: raise CommandException('"%s" command implementation is missing a ' 'command_spec definition.' % self.command_name) self.quiet_mode = not self.logger.isEnabledFor(logging.INFO) ui_controller = UIController(quiet_mode=self.quiet_mode, dump_status_messages_file=boto.config.get( 'GSUtil', 'dump_status_messages_file', None)) # Parse and validate args. self.args = self._TranslateDeprecatedAliases(args) self.ParseSubOpts() # Named tuple public functions start with _ # pylint: disable=protected-access self.command_spec = self.command_spec._replace( urls_start_arg=self._CalculateUrlsStartArg()) if (len(self.args) < self.command_spec.min_args or len(self.args) > self.command_spec.max_args): self.RaiseWrongNumberOfArgumentsException() if self.command_name not in self._commands_with_subcommands_and_subopts: self.CheckArguments() # Build the support and default maps from the command spec. support_map = { 'gs': self.command_spec.gs_api_support, 's3': [ApiSelector.XML] } default_map = { 'gs': self.command_spec.gs_default_api, 's3': ApiSelector.XML } self.gsutil_api_map = GsutilApiMapFactory.GetApiMap( self.gsutil_api_class_map_factory, support_map, default_map) self.project_id = None self.gsutil_api = CloudApiDelegator(self.bucket_storage_uri_class, self.gsutil_api_map, self.logger, MainThreadUIQueue( sys.stderr, ui_controller), debug=self.debug, http_headers=self.non_metadata_headers, trace_token=self.trace_token, perf_trace_token=self.perf_trace_token, user_project=self.user_project) # Cross-platform path to run gsutil binary. self.gsutil_cmd = '' # If running on Windows, invoke python interpreter explicitly. if IS_WINDOWS: self.gsutil_cmd += 'python ' # Add full path to gsutil to make sure we test the correct version. self.gsutil_path = gslib.GSUTIL_PATH self.gsutil_cmd += self.gsutil_path # We're treating recursion_requested like it's used by all commands, but # only some of the commands accept the -R option. if self.sub_opts: for o, unused_a in self.sub_opts: if o == '-r' or o == '-R': self.recursion_requested = True break self.multiprocessing_is_available = ( CheckMultiprocessingAvailableAndInit().is_available) def RaiseWrongNumberOfArgumentsException(self): """Raises exception for wrong number of arguments supplied to command.""" if len(self.args) < self.command_spec.min_args: tail_str = 's' if self.command_spec.min_args > 1 else '' message = ('The %s command requires at least %d argument%s.' % (self.command_name, self.command_spec.min_args, tail_str)) else: message = ('The %s command accepts at most %d arguments.' % (self.command_name, self.command_spec.max_args)) message += ' Usage:\n%s\nFor additional help run:\n gsutil help %s' % ( self.command_spec.usage_synopsis, self.command_name) raise CommandException(message) def RaiseInvalidArgumentException(self): """Raises exception for specifying an invalid argument to command.""" message = ('Incorrect option(s) specified. Usage:\n%s\n' 'For additional help run:\n gsutil help %s' % (self.command_spec.usage_synopsis, self.command_name)) raise CommandException(message) def ParseSubOpts(self, check_args=False, args=None, should_update_sub_opts_and_args=True): """Parses sub-opt args. Args: check_args: True to have CheckArguments() called after parsing. args: List of args. If None, self.args will be used. should_update_sub_opts_and_args: True if self.sub_opts and self.args should be updated with the values returned after parsing. Else return a tuple of sub_opts, args returned by getopt.getopt. This is done to allow this method to be called from get_gcloud_storage_args in which case we do not want to update self.sub_opts and self.args. Raises: RaiseInvalidArgumentException: Invalid args specified. """ if args is None: unparsed_args = self.args else: unparsed_args = args try: parsed_sub_opts, parsed_args = getopt.getopt( unparsed_args, self.command_spec.supported_sub_args, self.command_spec.supported_private_args or []) except getopt.GetoptError: self.RaiseInvalidArgumentException() if should_update_sub_opts_and_args: self.sub_opts, self.args = parsed_sub_opts, parsed_args if check_args: self.CheckArguments() else: if check_args: # This is just for sanity check. Only get_gcloud_storage_args will # call this method with should_update_sub_opts_and_args=False, and it # does not set check_args to True. raise TypeError('Requested to check arguments' ' but sub_opts and args have not been updated.') return parsed_sub_opts, parsed_args def CheckArguments(self): """Checks that command line arguments match the command_spec. Any commands in self._commands_with_subcommands_and_subopts are responsible for calling this method after handling initial parsing of their arguments. This prevents commands with sub-commands as well as options from breaking the parsing of getopt. TODO: Provide a function to parse commands and sub-commands more intelligently once we stop allowing the deprecated command versions. Raises: CommandException if the arguments don't match. """ if (not self.command_spec.file_url_ok and HaveFileUrls(self.args[self.command_spec.urls_start_arg:])): raise CommandException('"%s" command does not support "file://" URLs. ' 'Did you mean to use a gs:// URL?' % self.command_name) if (not self.command_spec.provider_url_ok and HaveProviderUrls(self.args[self.command_spec.urls_start_arg:])): raise CommandException('"%s" command does not support provider-only ' 'URLs.' % self.command_name) def WildcardIterator(self, url_string, all_versions=False): """Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. Returns: WildcardIterator for use by caller. """ return CreateWildcardIterator(url_string, self.gsutil_api, all_versions=all_versions, project_id=self.project_id, logger=self.logger) def GetSeekAheadGsutilApi(self): """Helper to instantiate a Cloud API instance for a seek-ahead iterator. This must be separate from the core command.gsutil_api instance for thread-safety, since other iterators typically use that instance and the SeekAheadIterator operates in parallel. Returns: Cloud API instance for use by the seek-ahead iterator. """ # This is initialized in Initialize(Multiprocessing\|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined if not self.seek_ahead_gsutil_api: self.seek_ahead_gsutil_api = CloudApiDelegator( self.bucket_storage_uri_class, self.gsutil_api_map, logging.getLogger('dummy'), glob_status_queue, debug=self.debug, http_headers=self.non_metadata_headers, trace_token=self.trace_token, perf_trace_token=self.perf_trace_token, user_project=self.user_project) return self.seek_ahead_gsutil_api def RunCommand(self): """Abstract function in base class. Subclasses must implement this. The return value of this function will be used as the exit status of the process, so subclass commands should return an integer exit code (0 for success, a value in [1,255] for failure). """ raise CommandException('Command %s is missing its RunCommand() ' 'implementation' % self.command_name) ############################################################ # Shared helper functions that depend on base class state. # ############################################################ # TODO: Refactor ACL functions to a different module and pass the # command object as state, as opposed to defining them as member functions # of the command class. def ApplyAclFunc(self, acl_func, acl_excep_handler, url_strs, object_fields=None): """Sets the standard or default object ACL depending on self.command_name. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. url_strs: URL strings on which to set ACL. object_fields: If present, list of object metadata fields to retrieve; if None, default name expansion iterator fields will be used. Raises: CommandException if an ACL could not be set. """ multi_threaded_url_args = [] urls = list(map(StorageUrlFromString, url_strs)) if (UrlsAreMixOfBucketsAndObjects(urls) and not self.recursion_requested): raise CommandException('Cannot operate on a mix of buckets and objects.') # Handle bucket ACL setting operations single-threaded, because # our threading machinery currently assumes it's working with objects # (name_expansion_iterator), and normally we wouldn't expect users to need # to set ACLs on huge numbers of buckets at once anyway. for url in urls: if url.IsCloudUrl() and url.IsBucket(): if self.recursion_requested: # If user specified -R option, convert any bucket args to bucket # wildcards (e.g., gs://bucket/), to prevent the operation from # being applied to the buckets themselves. url.object_name = '' multi_threaded_url_args.append(url.url_string) else: # Convert to a NameExpansionResult so we can re-use the threaded # function for the single-threaded implementation. RefType is unused. for blr in self.WildcardIterator( url.url_string).IterBuckets(bucket_fields=['id']): name_expansion_for_url = NameExpansionResult( source_storage_url=url, is_multi_source_request=False, is_multi_top_level_source_request=False, names_container=False, expanded_storage_url=blr.storage_url, expanded_result=None) acl_func(self, name_expansion_for_url) else: multi_threaded_url_args.append(url.url_string) if len(multi_threaded_url_args) >= 1: name_expansion_iterator = NameExpansionIterator( self.command_name, self.debug, self.logger, self.gsutil_api, multi_threaded_url_args, self.recursion_requested, all_versions=self.all_versions, continue_on_error=self.continue_on_error or self.parallel_operations, bucket_listing_fields=object_fields) seek_ahead_iterator = SeekAheadNameExpansionIterator( self.command_name, self.debug, self.GetSeekAheadGsutilApi(), multi_threaded_url_args, self.recursion_requested, all_versions=self.all_versions) # Perform requests in parallel (-m) mode, if requested, using # configured number of parallel processes and threads. Otherwise, # perform requests with sequential function calls in current process. self.Apply(acl_func, name_expansion_iterator, acl_excep_handler, fail_on_error=not self.continue_on_error, seek_ahead_iterator=seek_ahead_iterator) if not self.everything_set_okay and not self.continue_on_error: raise CommandException('ACLs for some objects could not be set.') def SetAclFunc(self, name_expansion_result, thread_state=None): """Sets the object ACL for the name_expansion_result provided. Args: name_expansion_result: NameExpansionResult describing the target object. thread_state: If present, use this gsutil Cloud API instance for the set. """ if thread_state: assert not self.def_acl gsutil_api = thread_state else: gsutil_api = self.gsutil_api op_string = 'default object ACL' if self.def_acl else 'ACL' url = name_expansion_result.expanded_storage_url self.logger.info('Setting %s on %s...', op_string, url) if (gsutil_api.GetApiSelector(url.scheme) == ApiSelector.XML and url.scheme != 'gs'): # If we are called with a non-google ACL model, we need to use the XML # passthrough. acl_arg should either be a canned ACL or an XML ACL. self._SetAclXmlPassthrough(url, gsutil_api) else: # Normal Cloud API path. acl_arg is a JSON ACL or a canned ACL. self._SetAclGsutilApi(url, gsutil_api) PutToQueueWithTimeout(gsutil_api.status_queue, MetadataMessage(message_time=time.time())) def _SetAclXmlPassthrough(self, url, gsutil_api): """Sets the ACL for the URL provided using the XML passthrough functions. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either an XML string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. Must support XML passthrough functions. """ orig_prefer_api = gsutil_api.prefer_api try: gsutil_api.prefer_api = ApiSelector.XML gsutil_api.XmlPassThroughSetAcl(self.acl_arg, url, canned=self.canned, def_obj_acl=self.def_acl, provider=url.scheme) except ServiceException as e: if self.continue_on_error: self.everything_set_okay = False self.logger.error(e) else: raise finally: gsutil_api.prefer_api = orig_prefer_api def _SetAclGsutilApi(self, url, gsutil_api): """Sets the ACL for the URL provided using the gsutil Cloud API. This function assumes that self.def_acl, self.canned, and self.continue_on_error are initialized, and that self.acl_arg is either a JSON string or a canned ACL string. Args: url: CloudURL to set the ACL on. gsutil_api: gsutil Cloud API to use for the ACL set. """ try: if url.IsBucket(): if self.def_acl: if self.canned: gsutil_api.PatchBucket(url.bucket_name, apitools_messages.Bucket(), canned_def_acl=self.acl_arg, provider=url.scheme, fields=['id']) else: def_obj_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.ObjectAccessControl) if not def_obj_acl: # Use a sentinel value to indicate a private (no entries) default # object ACL. def_obj_acl.append(PRIVATE_DEFAULT_OBJ_ACL) bucket_metadata = apitools_messages.Bucket( defaultObjectAcl=def_obj_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) else: if self.canned: gsutil_api.PatchBucket(url.bucket_name, apitools_messages.Bucket(), canned_acl=self.acl_arg, provider=url.scheme, fields=['id']) else: bucket_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.BucketAccessControl) bucket_metadata = apitools_messages.Bucket(acl=bucket_acl) gsutil_api.PatchBucket(url.bucket_name, bucket_metadata, provider=url.scheme, fields=['id']) else: # url.IsObject() if self.canned: gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, apitools_messages.Object(), provider=url.scheme, generation=url.generation, canned_acl=self.acl_arg) else: object_acl = AclTranslation.JsonToMessage( self.acl_arg, apitools_messages.ObjectAccessControl) object_metadata = apitools_messages.Object(acl=object_acl) gsutil_api.PatchObjectMetadata(url.bucket_name, url.object_name, object_metadata, provider=url.scheme, generation=url.generation) except ArgumentException as e: raise except ServiceException as e: if self.continue_on_error: self.everything_set_okay = False self.logger.error(e) else: raise def SetAclCommandHelper(self, acl_func, acl_excep_handler): """Sets ACLs on the self.args using the passed-in acl function. Args: acl_func: ACL function to be passed to Apply. acl_excep_handler: ACL exception handler to be passed to Apply. """ acl_arg = self.args[0] url_args = self.args[1:] # Disallow multi-provider setacl requests, because there are differences in # the ACL models. if not UrlsAreForSingleProvider(url_args): raise CommandException('"%s" command spanning providers not allowed.' % self.command_name) # Determine whether acl_arg names a file containing XML ACL text vs. the # string name of a canned ACL. if os.path.isfile(acl_arg): with codecs.open(acl_arg, 'r', UTF8) as f: acl_arg = f.read() self.canned = False else: # No file exists, so expect a canned ACL string. # validate=False because we allow wildcard urls. storage_uri = boto.storage_uri( url_args[0], debug=self.debug, validate=False, bucket_storage_uri_class=self.bucket_storage_uri_class) canned_acls = storage_uri.canned_acls() if acl_arg not in canned_acls: raise CommandException('Invalid canned ACL "%s".' % acl_arg) self.canned = True # Used to track if any ACLs failed to be set. self.everything_set_okay = True self.acl_arg = acl_arg self.ApplyAclFunc(acl_func, acl_excep_handler, url_args) if not self.everything_set_okay and not self.continue_on_error: raise CommandException('ACLs for some objects could not be set.') def _WarnServiceAccounts(self): """Warns service account users who have received an AccessDenied error. When one of the metadata-related commands fails due to AccessDenied, user must ensure that they are listed as an Owner in the API console. """ # Import this here so that the value will be set first in # gcs_oauth2_boto_plugin. # pylint: disable=g-import-not-at-top from gcs_oauth2_boto_plugin.oauth2_plugin import IS_SERVICE_ACCOUNT if IS_SERVICE_ACCOUNT: # This method is only called when canned ACLs are used, so the warning # definitely applies. self.logger.warning('\n'.join( textwrap.wrap( 'It appears that your service account has been denied access while ' 'attempting to perform a metadata operation. If you believe that you ' 'should have access to this metadata (i.e., if it is associated with ' 'your account), please make sure that your service account' 's email ' 'address is listed as an Owner in the Permissions tab of the API ' 'console. See "gsutil help creds" for further information.\n'))) def GetAndPrintAcl(self, url_str): """Prints the standard or default object ACL depending on self.command_name. Args: url_str: URL string to get ACL for. """ blr = self.GetAclCommandBucketListingReference(url_str) url = StorageUrlFromString(url_str) if (self.gsutil_api.GetApiSelector(url.scheme) == ApiSelector.XML and url.scheme != 'gs'): # Need to use XML passthrough. try: acl = self.gsutil_api.XmlPassThroughGetAcl(url, def_obj_acl=self.def_acl, provider=url.scheme) print(acl.to_xml()) except AccessDeniedException as _: self._WarnServiceAccounts() raise else: if self.command_name == 'defacl': acl = blr.root_object.defaultObjectAcl if not acl: self.logger.warn( 'No default object ACL present for %s. This could occur if ' 'the default object ACL is private, in which case objects ' 'created in this bucket will be readable only by their ' 'creators. It could also mean you do not have OWNER permission ' 'on %s and therefore do not have permission to read the ' 'default object ACL. It could also mean that %s has Bucket ' 'Policy Only enabled and therefore object ACLs and default ' 'object ACLs are disabled (see ' 'https://cloud.google.com/storage/docs/bucket-policy-only).', url_str, url_str, url_str) else: acl = blr.root_object.acl # Use the access controls api to check if the acl is actually empty or # if the user has 403 access denied or 400 invalid argument. if not acl: self._ListAccessControlsAcl(url) print(AclTranslation.JsonFromMessage(acl)) def _ListAccessControlsAcl(self, storage_url): """Returns either bucket or object access controls for a storage url. Args: storage_url: StorageUrl object representing the bucket or object. Returns: BucketAccessControls, ObjectAccessControls, or None if storage_url does not represent a cloud bucket or cloud object. Raises: ServiceException if there was an error in the request. """ if storage_url.IsBucket(): return self.gsutil_api.ListBucketAccessControls( storage_url.bucket_name, provider=storage_url.scheme) elif storage_url.IsObject(): return self.gsutil_api.ListObjectAccessControls( storage_url.bucket_name, storage_url.object_name, provider=storage_url.scheme) else: return None def GetAclCommandBucketListingReference(self, url_str): """Gets a single bucket listing reference for an acl get command. Args: url_str: URL string to get the bucket listing reference for. Returns: BucketListingReference for the URL string. Raises: CommandException if string did not result in exactly one reference. """ # We're guaranteed by caller that we have the appropriate type of url # string for the call (ex. we will never be called with an object string # by getdefacl) wildcard_url = StorageUrlFromString(url_str) if wildcard_url.IsObject(): plurality_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterObjects( bucket_listing_fields=['acl'])) else: # Bucket or provider. We call IterBuckets explicitly here to ensure that # the root object is populated with the acl. if self.command_name == 'defacl': bucket_fields = ['defaultObjectAcl'] else: bucket_fields = ['acl'] plurality_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterBuckets( bucket_fields=bucket_fields)) if plurality_iter.IsEmpty(): raise CommandException('No URLs matched') if plurality_iter.HasPlurality(): raise CommandException( '%s matched more than one URL, which is not allowed by the %s ' 'command' % (url_str, self.command_name)) return list(plurality_iter)[0] def GetSingleBucketUrlFromArg(self, arg, bucket_fields=None): """Gets a single bucket URL based on the command arguments. Args: arg: String argument to get bucket URL for. bucket_fields: Fields to populate for the bucket. Returns: (StorageUrl referring to a single bucket, Bucket metadata). Raises: CommandException if args did not match exactly one bucket. """ plurality_checkable_iterator = self.GetBucketUrlIterFromArg( arg, bucket_fields=bucket_fields) if plurality_checkable_iterator.HasPlurality(): raise CommandException('%s matched more than one URL, which is not\n' 'allowed by the %s command' % (arg, self.command_name)) blr = list(plurality_checkable_iterator)[0] return StorageUrlFromString(blr.url_string), blr.root_object def GetBucketUrlIterFromArg(self, arg, bucket_fields=None): """Gets a single bucket URL based on the command arguments. Args: arg: String argument to iterate over. bucket_fields: Fields to populate for the bucket. Returns: PluralityCheckableIterator over buckets. Raises: CommandException if iterator matched no buckets. """ arg_url = StorageUrlFromString(arg) if not arg_url.IsCloudUrl() or arg_url.IsObject(): raise CommandException('"%s" command must specify a bucket' % self.command_name) plurality_checkable_iterator = PluralityCheckableIterator( self.WildcardIterator(arg).IterBuckets(bucket_fields=bucket_fields)) if plurality_checkable_iterator.IsEmpty(): raise CommandException('No URLs matched') return plurality_checkable_iterator ###################### # Private functions. # ###################### def _ResetConnectionPool(self): # Each OS process needs to establish its own set of connections to # the server to avoid writes from different OS processes interleaving # onto the same socket (and garbling the underlying SSL session). # We ensure each process gets its own set of connections here by # reinitializing state that tracks connections. connection_pool = StorageUri.provider_pool if connection_pool: for i in connection_pool: connection_pool[i].connection.close() StorageUri.provider_pool = {} StorageUri.connection = None def _GetProcessAndThreadCount(self, process_count, thread_count, parallel_operations_override, print_macos_warning=True): """Determines the values of process_count and thread_count. These values are used for parallel operations. If we're not performing operations in parallel, then ignore existing values and use process_count = thread_count = 1. Args: process_count: A positive integer or None. In the latter case, we read the value from the .boto config file. thread_count: A positive integer or None. In the latter case, we read the value from the .boto config file. parallel_operations_override: Used to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. print_macos_warning: Print a warning about parallel processing on MacOS if true. Returns: (process_count, thread_count): The number of processes and threads to use, respectively. """ # Set OS process and python thread count as a function of options # and config. if self.parallel_operations or parallel_operations_override: if not process_count: process_count = boto.config.getint( 'GSUtil', 'parallel_process_count', gslib.commands.config.DEFAULT_PARALLEL_PROCESS_COUNT) if process_count < 1: raise CommandException('Invalid parallel_process_count "%d".' % process_count) if not thread_count: thread_count = boto.config.getint( 'GSUtil', 'parallel_thread_count', gslib.commands.config.DEFAULT_PARALLEL_THREAD_COUNT) if thread_count < 1: raise CommandException('Invalid parallel_thread_count "%d".' % thread_count) else: # If -m not specified, then assume 1 OS process and 1 Python thread. process_count = 1 thread_count = 1 should_prohibit_multiprocessing, os_name = ShouldProhibitMultiprocessing() if should_prohibit_multiprocessing and process_count > 1: raise CommandException('\n'.join( textwrap.wrap( ('It is not possible to set process_count > 1 on %s. Please ' 'update your config file(s) (located at %s) and set ' '"parallel_process_count = 1".') % (os_name, ', '.join(GetFriendlyConfigFilePaths()))))) is_main_thread = self.recursive_apply_level == 0 if print_macos_warning and os_name == 'macOS' and process_count > 1 and is_main_thread: self.logger.info( 'If you experience problems with multiprocessing on MacOS, they ' 'might be related to https://bugs.python.org/issue33725. You can ' 'disable multiprocessing by editing your .boto config or by adding ' 'the following flag to your command: ' '`-o "GSUtil:parallel_process_count=1"`. Note that multithreading is ' 'still available even if you disable multiprocessing.\n') self.logger.debug('process count: %d', process_count) self.logger.debug('thread count: %d', thread_count) return (process_count, thread_count) def _SetUpPerCallerState(self): """Set up the state for a caller id, corresponding to one Apply call.""" # pylint: disable=global-variable-undefined,global-variable-not-assigned # These variables are initialized in InitializeMultiprocessingVariables or # InitializeThreadingVariables global global_return_values_map, shared_vars_map, failure_count global caller_id_finished_count, shared_vars_list_map, total_tasks global need_pool_or_done_cond, call_completed_map, class_map global task_queues, caller_id_lock, caller_id_counter # Get a new caller ID. with caller_id_lock: caller_id_counter.Increment() caller_id = caller_id_counter.GetValue() # Create a copy of self with an incremented recursive level. This allows # the class to report its level correctly if the function called from it # also needs to call Apply. cls = copy.copy(self) cls.recursive_apply_level += 1 # Thread-safe loggers can't be pickled, so we will remove it here and # recreate it later in the WorkerThread. This is not a problem since any # logger with the same name will be treated as a singleton. cls.logger = None # Likewise, the default API connection(s) can't be pickled, but are unused # anyway as each thread gets its own API delegator. cls.gsutil_api = None cls.seek_ahead_gsutil_api = None class_map[caller_id] = cls total_tasks[caller_id] = -1 # -1 => the producer hasn't finished yet. call_completed_map[caller_id] = False caller_id_finished_count[caller_id] = 0 global_return_values_map[caller_id] = [] return caller_id def _CreateNewConsumerPool(self, num_processes, num_threads, status_queue): """Create a new pool of processes that call _ApplyThreads.""" processes = [] task_queue = _NewMultiprocessingQueue() task_queues.append(task_queue) current_max_recursive_level.Increment() if current_max_recursive_level.GetValue() > MAX_RECURSIVE_DEPTH: raise CommandException('Recursion depth of Apply calls is too great.') for _ in range(num_processes): recursive_apply_level = len(consumer_pools) p = multiprocessing_context.Process(target=self._ApplyThreads, args=(num_threads, num_processes, recursive_apply_level, status_queue)) p.daemon = True processes.append(p) _CryptoRandomAtFork() p.start() consumer_pool = _ConsumerPool(processes, task_queue) consumer_pools.append(consumer_pool) class ParallelOverrideReason(object): """Enum class to describe purpose of overriding parallel operations.""" # For the case when we use slice parallelism. SLICE = 'slice' # For the case when we run a helper Apply call (such as in the _DiffIterator # of rsync) and override to make the command go faster. SPEED = 'speed' # For when we run Apply calls in perfdiag. PERFDIAG = 'perfdiag' def Apply(self, func, args_iterator, exception_handler, shared_attrs=None, arg_checker=_UrlArgChecker, parallel_operations_override=None, process_count=None, thread_count=None, should_return_results=False, fail_on_error=False, seek_ahead_iterator=None): """Calls _Parallel/SequentialApply based on multiprocessing availability. Args: func: Function to call to process each argument. args_iterator: Iterable collection of arguments to be put into the work queue. exception_handler: Exception handler for WorkerThread class. shared_attrs: List of attributes to manage across sub-processes. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. parallel_operations_override: A string (see ParallelOverrideReason) describing the reason to override self.parallel_operations. This allows the caller to safely override the top-level flag for a single call. process_count: The number of processes to use. If not specified, then the configured default will be used. thread_count: The number of threads per process. If not specified, then the configured default will be used.. should_return_results: If true, then return the results of all successful calls to func in a list. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. Used only if multiple processes and/or threads are used. Returns: Results from spawned threads. """ # This is initialized in Initialize(Multiprocessing\|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global thread_stats # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined if shared_attrs: original_shared_vars_values = {} # We'll add these back in at the end. for name in shared_attrs: original_shared_vars_values[name] = getattr(self, name) # By setting this to 0, we simplify the logic for computing deltas. # We'll add it back after all of the tasks have been performed. setattr(self, name, 0) (process_count, thread_count) = self._GetProcessAndThreadCount( process_count, thread_count, parallel_operations_override) is_main_thread = (self.recursive_apply_level == 0 and self.sequential_caller_id == -1) if is_main_thread: # This initializes the initial performance summary parameters. LogPerformanceSummaryParams(num_processes=process_count, num_threads=thread_count) # We don't honor the fail_on_error flag in the case of multiple threads # or processes. fail_on_error = fail_on_error and (process_count * thread_count == 1) # Only check this from the first call in the main thread. Apart from the # fact that it's wasteful to try this multiple times in general, it also # will never work when called from a subprocess since we use daemon # processes, and daemons can't create other processes. if (is_main_thread and not self.multiprocessing_is_available and process_count > 1): # Run the check again and log the appropriate warnings. This was run # before, when the Command object was created, in order to calculate # self.multiprocessing_is_available, but we don't want to print the # warning until we're sure the user actually tried to use multiple # threads or processes. CheckMultiprocessingAvailableAndInit(logger=self.logger) caller_id = self._SetUpPerCallerState() # If any shared attributes passed by caller, create a dictionary of # shared memory variables for every element in the list of shared # attributes. if shared_attrs: shared_vars_list_map[caller_id] = shared_attrs for name in shared_attrs: shared_vars_map[(caller_id, name)] = 0 # Make all of the requested function calls. usable_processes_count = (process_count if self.multiprocessing_is_available else 1) if thread_count * usable_processes_count > 1: self._ParallelApply( func, args_iterator, exception_handler, caller_id, arg_checker, usable_processes_count, thread_count, should_return_results, fail_on_error, seek_ahead_iterator=seek_ahead_iterator, parallel_operations_override=parallel_operations_override) if is_main_thread: _AggregateThreadStats() else: self._SequentialApply(func, args_iterator, exception_handler, caller_id, arg_checker, should_return_results, fail_on_error) if shared_attrs: for name in shared_attrs: # This allows us to retain the original value of the shared variable, # and simply apply the delta after what was done during the call to # apply. final_value = (original_shared_vars_values[name] + shared_vars_map.get( (caller_id, name))) setattr(self, name, final_value) if should_return_results: return global_return_values_map.get(caller_id) def _MaybeSuggestGsutilDashM(self): """Outputs a suggestion to the user to use gsutil -m.""" if not (boto.config.getint('GSUtil', 'parallel_process_count', 0) == 1 and boto.config.getint('GSUtil', 'parallel_thread_count', 0) == 1): self.logger.info('\n' + textwrap.fill( '==> NOTE: You are performing a sequence of gsutil operations that ' 'may run significantly faster if you instead use gsutil -m %s ...\n' 'Please see the -m section under "gsutil help options" for further ' 'information about when gsutil -m can be advantageous.' % self.command_spec.command_name) + '\n') # pylint: disable=g-doc-args def _SequentialApply(self, func, args_iterator, exception_handler, caller_id, arg_checker, should_return_results, fail_on_error): """Performs all function calls sequentially in the current thread. No other threads or processes will be spawned. This degraded functionality is used when the multiprocessing module is not available or the user requests only one thread and one process. """ # Create a WorkerThread to handle all of the logic needed to actually call # the function. Note that this thread will never be started, and all work # is done in the current thread. worker_thread = WorkerThread(None, False, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) args_iterator = iter(args_iterator) # Count of sequential calls that have been made. Used for producing # suggestion to use gsutil -m. sequential_call_count = 0 while True: # Try to get the next argument, handling any exceptions that arise. try: args = next(args_iterator) except StopIteration as e: break except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if fail_on_error: raise else: try: exception_handler(self, e) except Exception as _: # pylint: disable=broad-except self.logger.debug( 'Caught exception while handling exception for %s:\n%s', func, traceback.format_exc()) continue sequential_call_count += 1 if (sequential_call_count == OFFER_GSUTIL_M_SUGGESTION_THRESHOLD or sequential_call_count % OFFER_GSUTIL_M_SUGGESTION_FREQUENCY == 0): # Output suggestion near beginning of run, so user sees it early, and # every so often while the command is executing, so they can ^C and try # gsutil -m. self._MaybeSuggestGsutilDashM() if arg_checker(self, args): # Now that we actually have the next argument, perform the task. task = Task(func, args, caller_id, exception_handler, should_return_results, arg_checker, fail_on_error) worker_thread.PerformTask(task, self) lines_since_suggestion_last_printed = (sequential_call_count % OFFER_GSUTIL_M_SUGGESTION_FREQUENCY) if lines_since_suggestion_last_printed >= GetTermLines(): # Output suggestion at end of long run, in case user missed it at the # start and it scrolled off-screen. self._MaybeSuggestGsutilDashM() PutToQueueWithTimeout(self.gsutil_api.status_queue, FinalMessage(time.time())) # If the final iterated argument results in an exception, and that # exception modifies shared_attrs, we need to publish the results. worker_thread.shared_vars_updater.Update(caller_id, self) # Now that all the work is done, log the types of source URLs encountered. self._ProcessSourceUrlTypes(args_iterator) # pylint: disable=g-doc-args def _ParallelApply(self, func, args_iterator, exception_handler, caller_id, arg_checker, process_count, thread_count, should_return_results, fail_on_error, seek_ahead_iterator=None, parallel_operations_override=None): r"""Dispatches input arguments across a thread/process pool. Pools are composed of parallel OS processes and/or Python threads, based on options (-m or not) and settings in the user's config file. If only one OS process is requested/available, dispatch requests across threads in the current OS process. In the multi-process case, we will create one pool of worker processes for each level of the tree of recursive calls to Apply. E.g., if A calls Apply(B), and B ultimately calls Apply(C) followed by Apply(D), then we will only create two sets of worker processes - B will execute in the first, and C and D will execute in the second. If C is then changed to call Apply(E) and D is changed to call Apply(F), then we will automatically create a third set of processes (lazily, when needed) that will be used to execute calls to E and F. This might look something like: Pool1 Executes: B / \ Pool2 Executes: C D / \ Pool3 Executes: E F Apply's parallelism is generally broken up into 4 cases: - If process_count == thread_count == 1, then all tasks will be executed by _SequentialApply. - If process_count > 1 and thread_count == 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will execute the tasks in a single thread. - If process_count == 1 and thread_count > 1, then this process will create a new pool of threads to execute the tasks. - If process_count > 1 and thread_count > 1, then the main thread will create a new pool of processes (if they don't already exist) and each of those processes will, upon creation, create a pool of threads to execute the tasks. Args: caller_id: The caller ID unique to this call to command.Apply. See command.Apply for description of other arguments. """ # This is initialized in Initialize(Multiprocessing\|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue, ui_controller # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined is_main_thread = self.recursive_apply_level == 0 if (parallel_operations_override == self.ParallelOverrideReason.SLICE and self.recursive_apply_level <= 1): # The operation uses slice parallelism if the recursive apply level > 0 or # if we're executing _ParallelApply without the -m option. glob_status_queue.put(PerformanceSummaryMessage(time.time(), True)) if not IS_WINDOWS and is_main_thread: # For multi-thread or multi-process scenarios, the main process must # kill itself on a terminating signal, because sys.exit(1) only exits # the currently executing thread, leaving orphaned processes. The main # thread is responsible for cleaning up multiprocessing variables such # as manager processes. Therefore, the main thread's signal handling # chain is: # 1: __main__._CleanupSignalHandler (clean up processes) # 2: MultithreadedSignalHandler (kill self) for signal_num in (signal.SIGINT, signal.SIGTERM): RegisterSignalHandler(signal_num, MultithreadedMainSignalHandler, is_final_handler=True) if not task_queues: # The process we create will need to access the next recursive level # of task queues if it makes a call to Apply, so we always keep around # one more queue than we know we need. OTOH, if we don't create a new # process, the existing process still needs a task queue to use. if process_count > 1: task_queues.append(_NewMultiprocessingQueue()) else: task_queue = _NewThreadsafeQueue() task_queues.append(task_queue) # Create a top-level worker pool since this is the first execution # of ParallelApply on the main thread. WorkerPool(thread_count, self.logger, task_queue=task_queue, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=glob_status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) if process_count > 1: # Handle process pool creation. # Check whether this call will need a new set of workers. # Each worker must acquire a shared lock before notifying the main thread # that it needs a new worker pool, so that at most one worker asks for # a new worker pool at once. try: if not is_main_thread: worker_checking_level_lock.acquire() if self.recursive_apply_level >= current_max_recursive_level.GetValue(): with need_pool_or_done_cond: # Only the main thread is allowed to create new processes - # otherwise, we will run into some Python bugs. if is_main_thread: self._CreateNewConsumerPool(process_count, thread_count, glob_status_queue) else: # Notify the main thread that we need a new consumer pool. new_pool_needed.Reset(reset_value=1) need_pool_or_done_cond.notify_all() # The main thread will notify us when it finishes. need_pool_or_done_cond.wait() finally: if not is_main_thread: worker_checking_level_lock.release() else: # Handle new worker thread pool creation. if not is_main_thread: try: worker_checking_level_lock.acquire() if self.recursive_apply_level > _GetCurrentMaxRecursiveLevel(): # We don't have a thread pool for this level of recursive apply # calls, so create a pool and corresponding task queue. _IncrementCurrentMaxRecursiveLevel() task_queue = _NewThreadsafeQueue() task_queues.append(task_queue) WorkerPool(thread_count, self.logger, task_queue=task_queue, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=glob_status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) finally: worker_checking_level_lock.release() task_queue = task_queues[self.recursive_apply_level] # Only use the seek-ahead iterator in the main thread to provide an # overall estimate of operations. if seek_ahead_iterator and not is_main_thread: seek_ahead_iterator = None # Kick off a producer thread to throw tasks in the global task queue. We # do this asynchronously so that the main thread can be free to create new # consumer pools when needed (otherwise, any thread with a task that needs # a new consumer pool must block until we're completely done producing; in # the worst case, every worker blocks on such a call and the producer fills # up the task queue before it finishes, so we block forever). producer_thread = ProducerThread( copy.copy(self), args_iterator, caller_id, func, task_queue, should_return_results, exception_handler, arg_checker, fail_on_error, seek_ahead_iterator=seek_ahead_iterator, status_queue=(glob_status_queue if is_main_thread else None)) # Start the UI thread that is responsible for displaying operation status # (aggregated across processes and threads) to the user. ui_thread = None if is_main_thread: ui_thread = UIThread(glob_status_queue, sys.stderr, ui_controller) # Wait here until either: # 1. We're the main thread in the multi-process case, and someone needs # a new consumer pool - in which case we create one and continue # waiting. # 2. Someone notifies us that all of the work we requested is done, in # which case we retrieve the results (if applicable) and stop # waiting. # At most one of these can be true, because the main thread is blocked on # its call to Apply, and a thread will not ask for a new consumer pool # unless it had more work to do. while True: with need_pool_or_done_cond: if call_completed_map[caller_id]: break elif (process_count > 1 and is_main_thread and new_pool_needed.GetValue()): new_pool_needed.Reset() self._CreateNewConsumerPool(process_count, thread_count, glob_status_queue) need_pool_or_done_cond.notify_all() # Note that we must check the above conditions before the wait() call; # otherwise, the notification can happen before we start waiting, in # which case we'll block forever. need_pool_or_done_cond.wait() # We've completed all tasks (or excepted), so signal the UI thread to # terminate. if is_main_thread: PutToQueueWithTimeout(glob_status_queue, ZERO_TASKS_TO_DO_ARGUMENT) ui_thread.join(timeout=UI_THREAD_JOIN_TIMEOUT) # Now that all the work is done, log the types of source URLs encountered. self._ProcessSourceUrlTypes(producer_thread.args_iterator) # We encountered an exception from the producer thread before any arguments # were enqueued, but it wouldn't have been propagated, so we'll now # explicitly raise it here. if producer_thread.unknown_exception: # pylint: disable=raising-bad-type raise producer_thread.unknown_exception # We encountered an exception from the producer thread while iterating over # the arguments, so raise it here if we're meant to fail on error. if producer_thread.iterator_exception and fail_on_error: # pylint: disable=raising-bad-type raise producer_thread.iterator_exception if is_main_thread and not parallel_operations_override: PutToQueueWithTimeout(glob_status_queue, FinalMessage(time.time())) def _ProcessSourceUrlTypes(self, args_iterator): """Logs the URL type information to analytics collection.""" if not isinstance(args_iterator, CopyObjectsIterator): return LogPerformanceSummaryParams(is_daisy_chain=args_iterator.is_daisy_chain, has_file_src=args_iterator.has_file_src, has_cloud_src=args_iterator.has_cloud_src, provider_types=args_iterator.provider_types) def _ApplyThreads(self, thread_count, process_count, recursive_apply_level, status_queue): """Assigns the work from the multi-process global task queue. Work is assigned to an individual process for later consumption either by the WorkerThreads or (if thread_count == 1) this thread. Args: thread_count: The number of threads used to perform the work. If 1, then perform all work in this thread. process_count: The number of processes used to perform the work. recursive_apply_level: The depth in the tree of recursive calls to Apply of this thread. status_queue: Multiprocessing/threading queue for progress reporting and performance aggregation. """ assert process_count > 1, ( 'Invalid state, calling command._ApplyThreads with only one process.') _CryptoRandomAtFork() # Separate processes should exit on a terminating signal, # but to avoid race conditions only the main process should handle # multiprocessing cleanup. Override child processes to use a single signal # handler. for catch_signal in GetCaughtSignals(): signal.signal(catch_signal, ChildProcessSignalHandler) self._ResetConnectionPool() self.recursive_apply_level = recursive_apply_level task_queue = task_queues[recursive_apply_level] # Ensure fairness across processes by filling our WorkerPool # only with as many tasks as it has WorkerThreads. This semaphore is # acquired each time that a task is retrieved from the queue and released # each time a task is completed by a WorkerThread. worker_semaphore = threading.BoundedSemaphore(thread_count) # TODO: Presently, this pool gets recreated with each call to Apply. We # should be able to do it just once, at process creation time. worker_pool = WorkerPool( thread_count, self.logger, worker_semaphore=worker_semaphore, bucket_storage_uri_class=self.bucket_storage_uri_class, gsutil_api_map=self.gsutil_api_map, debug=self.debug, status_queue=status_queue, headers=self.non_metadata_headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) num_enqueued = 0 while True: while not worker_semaphore.acquire(blocking=False): # Because Python signal handlers are only called in between atomic # instructions, if we block the main thread on an available worker # thread, we won't be able to respond to signals such as a # user-initiated CTRL-C until a worker thread completes a task. # We poll the semaphore periodically as a compromise between # efficiency and user responsiveness. time.sleep(0.01) task = task_queue.get() if task.args != ZERO_TASKS_TO_DO_ARGUMENT: # If we have no tasks to do and we're performing a blocking call, we # need a special signal to tell us to stop - otherwise, we block on # the call to task_queue.get() forever. worker_pool.AddTask(task) num_enqueued += 1 else: # No tasks remain; since no work was dispatched to a thread, don't # block the semaphore on a WorkerThread completion. worker_semaphore.release() # Below here lie classes and functions related to controlling the flow of tasks # between various threads and processes. class _ConsumerPool(object): def __init__(self, processes, task_queue): self.processes = processes self.task_queue = task_queue def ShutDown(self): for process in self.processes: KillProcess(process.pid) class Task( namedtuple('Task', ( 'func args caller_id exception_handler should_return_results arg_checker ' 'fail_on_error'))): """Task class representing work to be completed. Args: func: The function to be executed. args: The arguments to func. caller_id: The globally-unique caller ID corresponding to the Apply call. exception_handler: The exception handler to use if the call to func fails. should_return_results: True iff the results of this function should be returned from the Apply call. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. """ pass # TODO: Refactor the various threading code that doesn't need to depend on # command.py globals (ProducerThread, UIThread) to different files to aid # readability and reduce the size of command.py. def _StartSeekAheadThread(seek_ahead_iterator, seek_ahead_thread_cancel_event): """Initializes and runs the seek-ahead thread. We defer starting this thread until it is needed, since it is only useful when the ProducerThread iterates more results than it can store on the global task queue. Args: seek_ahead_iterator: Iterator that yields SeekAheadResults. seek_ahead_thread_cancel_event: threading.Event for signaling the seek-ahead thread to terminate. Returns: The thread object for the initialized thread. """ # This is initialized in Initialize(Multiprocessing\|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global glob_status_queue # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined return SeekAheadThread(seek_ahead_iterator, seek_ahead_thread_cancel_event, glob_status_queue) class ProducerThread(threading.Thread): """Thread used to enqueue work for other processes and threads.""" def __init__(self, cls, args_iterator, caller_id, func, task_queue, should_return_results, exception_handler, arg_checker, fail_on_error, seek_ahead_iterator=None, status_queue=None): """Initializes the producer thread. Args: cls: Instance of Command for which this ProducerThread was created. args_iterator: Iterable collection of arguments to be put into the work queue. caller_id: Globally-unique caller ID corresponding to this call to Apply. func: The function to be called on each element of args_iterator. task_queue: The queue into which tasks will be put, to later be consumed by Command._ApplyThreads. should_return_results: True iff the results for this call to command.Apply were requested. exception_handler: The exception handler to use when errors are encountered during calls to func. arg_checker: Used to determine whether we should process the current argument or simply skip it. Also handles any logging that is specific to a particular type of argument. fail_on_error: If true, then raise any exceptions encountered when executing func. This is only applicable in the case of process_count == thread_count == 1. seek_ahead_iterator: If present, a seek-ahead iterator that will provide an approximation of the total number of tasks and bytes that will be iterated by the ProducerThread. status_queue: status_queue to inform task_queue estimation. Only valid when calling from the main thread, else None. Even if this is the main thread, the status_queue will only properly work if args is a collection of NameExpansionResults, which is the type that gives us initial information about files to be processed. Otherwise, nothing will be added to the queue. """ super(ProducerThread, self).__init__() self.func = func self.cls = cls self.args_iterator = args_iterator self.caller_id = caller_id self.task_queue = task_queue self.arg_checker = arg_checker self.exception_handler = exception_handler self.should_return_results = should_return_results self.fail_on_error = fail_on_error self.shared_variables_updater = _SharedVariablesUpdater() self.daemon = True self.unknown_exception = None self.iterator_exception = None self.seek_ahead_iterator = seek_ahead_iterator self.status_queue = status_queue self.start() def run(self): num_tasks = 0 cur_task = None last_task = None task_estimation_threshold = None seek_ahead_thread = None seek_ahead_thread_cancel_event = None seek_ahead_thread_considered = False args = None try: total_size = 0 self.args_iterator = iter(self.args_iterator) while True: try: args = next(self.args_iterator) except StopIteration as e: break except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if self.fail_on_error: self.iterator_exception = e raise else: try: self.exception_handler(self.cls, e) except Exception as _: # pylint: disable=broad-except self.cls.logger.debug( 'Caught exception while handling exception for %s:\n%s', self.func, traceback.format_exc()) self.shared_variables_updater.Update(self.caller_id, self.cls) continue if self.arg_checker(self.cls, args): num_tasks += 1 if self.status_queue: if not num_tasks % 100: # Time to update the total number of tasks. if (isinstance(args, NameExpansionResult) or isinstance(args, CopyObjectInfo) or isinstance(args, RsyncDiffToApply)): PutToQueueWithTimeout( self.status_queue, ProducerThreadMessage(num_tasks, total_size, time.time())) if (isinstance(args, NameExpansionResult) or isinstance(args, CopyObjectInfo)): if args.expanded_result: json_expanded_result = json.loads(args.expanded_result) if 'size' in json_expanded_result: total_size += int(json_expanded_result['size']) elif isinstance(args, RsyncDiffToApply): if args.copy_size: total_size += int(args.copy_size) if not seek_ahead_thread_considered: if task_estimation_threshold is None: task_estimation_threshold = _GetTaskEstimationThreshold() if task_estimation_threshold <= 0: # Disable the seek-ahead thread (never start it). seek_ahead_thread_considered = True elif num_tasks >= task_estimation_threshold: if self.seek_ahead_iterator: seek_ahead_thread_cancel_event = threading.Event() seek_ahead_thread = _StartSeekAheadThread( self.seek_ahead_iterator, seek_ahead_thread_cancel_event) # For integration testing only, force estimation to complete # prior to producing further results. if boto.config.get('GSUtil', 'task_estimation_force', None): seek_ahead_thread.join(timeout=SEEK_AHEAD_JOIN_TIMEOUT) seek_ahead_thread_considered = True last_task = cur_task cur_task = Task(self.func, args, self.caller_id, self.exception_handler, self.should_return_results, self.arg_checker, self.fail_on_error) if last_task: self.task_queue.put(last_task) except Exception as e: # pylint: disable=broad-except # This will also catch any exception raised due to an error in the # iterator when fail_on_error is set, so check that we failed for some # other reason before claiming that we had an unknown exception. if not self.iterator_exception: self.unknown_exception = e finally: # We need to make sure to update total_tasks[caller_id] before we enqueue # the last task. Otherwise, a worker can retrieve the last task and # complete it, then check total_tasks and determine that we're not done # producing all before we update total_tasks. This approach forces workers # to wait on the last task until after we've updated total_tasks. total_tasks[self.caller_id] = num_tasks if not cur_task: # This happens if there were zero arguments to be put in the queue. cur_task = Task(None, ZERO_TASKS_TO_DO_ARGUMENT, self.caller_id, None, None, None, None) self.task_queue.put(cur_task) # If the seek ahead thread is still running, cancel it and wait for it # to exit since we've enumerated all of the tasks already. We don't want # to delay command completion on an estimate that has become meaningless. if seek_ahead_thread is not None: seek_ahead_thread_cancel_event.set() # It's possible that the seek-ahead-thread may attempt to put to the # status queue after it has been torn down, for example if the system # is overloaded. Because the put uses a timeout, it should never block # command termination or signal handling. seek_ahead_thread.join(timeout=SEEK_AHEAD_JOIN_TIMEOUT) # Send a final ProducerThread message that definitively states # the amount of actual work performed. if (self.status_queue and (isinstance(args, NameExpansionResult) or isinstance( args, CopyObjectInfo) or isinstance(args, RsyncDiffToApply))): PutToQueueWithTimeout( self.status_queue, ProducerThreadMessage(num_tasks, total_size, time.time(), finished=True)) # It's possible that the workers finished before we updated total_tasks, # so we need to check here as well. _NotifyIfDone(self.caller_id, caller_id_finished_count.get(self.caller_id)) class WorkerPool(object): """Pool of worker threads to which tasks can be added.""" def __init__(self, thread_count, logger, worker_semaphore=None, task_queue=None, bucket_storage_uri_class=None, gsutil_api_map=None, debug=0, status_queue=None, headers=None, perf_trace_token=None, trace_token=None, user_project=None): # In the multi-process case, a worker sempahore is required to ensure # even work distribution. # # In the single process case, the input task queue directly feeds worker # threads from the ProducerThread. Since worker threads will consume only # one task at a time from the queue, there is no need for a semaphore to # ensure even work distribution. # # Thus, exactly one of task_queue or worker_semaphore must be provided. assert (worker_semaphore is None) != (task_queue is None) self.headers = headers self.perf_trace_token = perf_trace_token self.trace_token = trace_token self.user_project = user_project self.task_queue = task_queue or _NewThreadsafeQueue() self.threads = [] for _ in range(thread_count): worker_thread = WorkerThread( self.task_queue, logger, worker_semaphore=worker_semaphore, bucket_storage_uri_class=bucket_storage_uri_class, gsutil_api_map=gsutil_api_map, debug=debug, status_queue=status_queue, headers=self.headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) self.threads.append(worker_thread) worker_thread.start() def AddTask(self, task): """Adds a task to the task queue; used only in the multi-process case.""" self.task_queue.put(task) class WorkerThread(threading.Thread): """Thread where all the work will be performed. This makes the function calls for Apply and takes care of all error handling, return value propagation, and shared_vars. Note that this thread is NOT started upon instantiation because the function- calling logic is also used in the single-threaded case. """ # This is initialized in Initialize(Multiprocessing\|Threading)Variables # pylint: disable=global-variable-not-assigned # pylint: disable=global-variable-undefined global thread_stats # pylint: enable=global-variable-not-assigned # pylint: enable=global-variable-undefined def __init__(self, task_queue, logger, worker_semaphore=None, bucket_storage_uri_class=None, gsutil_api_map=None, debug=0, status_queue=None, headers=None, perf_trace_token=None, trace_token=None, user_project=None): """Initializes the worker thread. Args: task_queue: The thread-safe queue from which this thread should obtain its work. logger: Logger to use for this thread. worker_semaphore: threading.BoundedSemaphore to be released each time a task is completed, or None for single-threaded execution. bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_map: Map of providers and API selector tuples to api classes which can be used to communicate with those providers. Used for the instantiating CloudApiDelegator class. debug: debug level for the CloudApiDelegator class. status_queue: Queue for reporting status updates. user_project: Project to be billed for this request. """ super(WorkerThread, self).__init__() self.pid = os.getpid() self.init_time = time.time() self.task_queue = task_queue self.worker_semaphore = worker_semaphore self.daemon = True self.cached_classes = {} self.shared_vars_updater = _SharedVariablesUpdater() self.headers = headers self.perf_trace_token = perf_trace_token self.trace_token = trace_token self.user_project = user_project # Note that thread_gsutil_api is not initialized in the sequential # case; task functions should use utils.cloud_api_helper.GetCloudApiInstance # to retrieve the main thread's CloudApiDelegator in that case. self.thread_gsutil_api = None if bucket_storage_uri_class and gsutil_api_map: self.thread_gsutil_api = CloudApiDelegator( bucket_storage_uri_class, gsutil_api_map, logger, status_queue, debug=debug, http_headers=self.headers, perf_trace_token=self.perf_trace_token, trace_token=self.trace_token, user_project=self.user_project) @CaptureThreadStatException def _StartBlockedTime(self): """Update the thread_stats AtomicDict before task_queue.get() is called.""" if thread_stats.get((self.pid, self.ident)) is None: thread_stats[(self.pid, self.ident)] = _ThreadStat(self.init_time) # While this read/modify/write is not an atomic operation on the dict, # we are protected since the (process ID, thread ID) tuple is unique # to this thread, making this thread the only reader/writer for this key. thread_stat = thread_stats[(self.pid, self.ident)] thread_stat.StartBlockedTime() thread_stats[(self.pid, self.ident)] = thread_stat @CaptureThreadStatException def _EndBlockedTime(self): """Update the thread_stats AtomicDict after task_queue.get() is called.""" thread_stat = thread_stats[(self.pid, self.ident)] thread_stat.EndBlockedTime() thread_stats[(self.pid, self.ident)] = thread_stat def PerformTask(self, task, cls): """Makes the function call for a task. Args: task: The Task to perform. cls: The instance of a class which gives context to the functions called by the Task's function. E.g., see SetAclFuncWrapper. """ caller_id = task.caller_id try: results = task.func(cls, task.args, thread_state=self.thread_gsutil_api) if task.should_return_results: global_return_values_map.Increment(caller_id, [results], default_value=[]) except Exception as e: # pylint: disable=broad-except _IncrementFailureCount() if task.fail_on_error: raise # Only happens for single thread and process case. else: try: task.exception_handler(cls, e) except Exception as _: # pylint: disable=broad-except # Don't allow callers to raise exceptions here and kill the worker # threads. cls.logger.debug( 'Caught exception while handling exception for %s:\n%s', task, traceback.format_exc()) finally: if self.worker_semaphore: self.worker_semaphore.release() self.shared_vars_updater.Update(caller_id, cls) # Even if we encounter an exception, we still need to claim that that # the function finished executing. Otherwise, we won't know when to # stop waiting and return results. num_done = caller_id_finished_count.Increment(caller_id, 1) _NotifyIfDone(caller_id, num_done) def run(self): while True: self._StartBlockedTime() task = self.task_queue.get() self._EndBlockedTime() if task.args == ZERO_TASKS_TO_DO_ARGUMENT: # This can happen in the single-process case because worker threads # consume ProducerThread tasks directly. continue caller_id = task.caller_id # Get the instance of the command with the appropriate context. cls = self.cached_classes.get(caller_id, None) if not cls: cls = copy.copy(class_map[caller_id]) cls.logger = CreateOrGetGsutilLogger(cls.command_name) self.cached_classes[caller_id] = cls self.PerformTask(task, cls) class _ThreadStat(object): """Stores thread idle and execution time statistics.""" def __init__(self, init_time): self.total_idle_time = 0 # The last time EndBlockedTime was called, which is the last time a # task_queue.get() completed or when we initialized the thread. self.end_block_time = init_time # The last time StartBlockedTime was called, which is the last time a # task_queue.get() call started. self.start_block_time = time.time() # Between now and thread initialization, we were not blocked. self.total_execution_time = 0 def StartBlockedTime(self): self.start_block_time = time.time() exec_time = self.start_block_time - self.end_block_time self.total_execution_time += exec_time def EndBlockedTime(self): self.end_block_time = time.time() idle_time = self.end_block_time - self.start_block_time self.total_idle_time += idle_time def AggregateStat(self, end_time): """Decide final stats upon Apply completion.""" if self.end_block_time > self.start_block_time: # Apply ended before we blocked on task_queue.get(), or there was an # exception during StartBlockedTime. In both of these cases, we were not # blocked on task_queue.get() and so can add this time to execution time. self.total_execution_time += end_time - self.end_block_time else: # Apply ended while we were blocked on task_queue.get(), or there was an # exception during EndBlockedTime. In both of these cases, we were in the # midst of or just finishing a task_queue.get() call, and so can add this # time to idle time. self.total_idle_time += end_time - self.start_block_time def _AggregateThreadStats(): """At the end of the top-level Apply call, aggregate the thread stats dict. This should only be called in the main process and thread because it logs to the MetricsCollector. """ cur_time = time.time() total_idle_time = total_execution_time = 0 for thread_stat in thread_stats.values(): thread_stat.AggregateStat(cur_time) total_idle_time += thread_stat.total_idle_time total_execution_time += thread_stat.total_execution_time LogPerformanceSummaryParams(thread_idle_time=total_idle_time, thread_execution_time=total_execution_time) class _SharedVariablesUpdater(object): """Used to update shared variable for a class in the global map. Note that each thread will have its own instance of the calling class for context, and it will also have its own instance of a _SharedVariablesUpdater. This is used in the following way: 1. Before any tasks are performed, each thread will get a copy of the calling class, and the globally-consistent value of this shared variable will be initialized to whatever it was before the call to Apply began. 2. After each time a thread performs a task, it will look at the current values of the shared variables in its instance of the calling class. 2.A. For each such variable, it computes the delta of this variable between the last known value for this class (which is stored in a dict local to this class) and the current value of the variable in the class. 2.B. Using this delta, we update the last known value locally as well as the globally-consistent value shared across all classes (the globally consistent value is simply increased by the computed delta). """ def __init__(self): self.last_shared_var_values = {} def Update(self, caller_id, cls): """Update any shared variables with their deltas.""" shared_vars = shared_vars_list_map.get(caller_id, None) if shared_vars: for name in shared_vars: key = (caller_id, name) last_value = self.last_shared_var_values.get(key, 0) # Compute the change made since the last time we updated here. This is # calculated by simply subtracting the last known value from the current # value in the class instance. delta = getattr(cls, name) - last_value self.last_shared_var_values[key] = delta + last_value # Update the globally-consistent value by simply increasing it by the # computed delta. shared_vars_map.Increment(key, delta) def _NotifyIfDone(caller_id, num_done): """Notify any threads waiting for results that something has finished. Each waiting thread will then need to check the call_completed_map to see if its work is done. Note that num_done could be calculated here, but it is passed in as an optimization so that we have one less call to a globally-locked data structure. Args: caller_id: The caller_id of the function whose progress we're checking. num_done: The number of tasks currently completed for that caller_id. """ num_to_do = total_tasks[caller_id] if num_to_do == num_done and num_to_do >= 0: # Notify the Apply call that's sleeping that it's ready to return. with need_pool_or_done_cond: call_completed_map[caller_id] = True need_pool_or_done_cond.notify_all() # pylint: disable=global-variable-not-assigned,global-variable-undefined def ShutDownGsutil(): """Shut down all processes in consumer pools in preparation for exiting.""" global glob_status_queue for q in queues: try: q.cancel_join_thread() except: # pylint: disable=bare-except pass for consumer_pool in consumer_pools: consumer_pool.ShutDown() try: glob_status_queue.cancel_join_thread() except: # pylint: disable=bare-except pass def _GetCurrentMaxRecursiveLevel(): global current_max_recursive_level return current_max_recursive_level.GetValue() def _IncrementCurrentMaxRecursiveLevel(): global current_max_recursive_level current_max_recursive_level.Increment() def _IncrementFailureCount(): global failure_count failure_count.Increment() def DecrementFailureCount(): global failure_count failure_count.Decrement() def GetFailureCount(): """Returns the number of failures processed during calls to Apply().""" global failure_count return failure_count.GetValue() def ResetFailureCount(): """Resets the failure_count variable to 0 - useful if error is expected.""" global failure_count failure_count.Reset()	110.58 KB	2025-06-25 08:19
📄 command_argument.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Contains classes related to argparse-based argument parsing.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from .tab_complete import CompleterType class CommandArgument(object): """Argparse style argument.""" def __init__(self, args, kwargs): """Constructs an argparse argument with the given data. See add_argument in argparse for description of the options. The only deviation from the argparse arguments is the 'completer' parameter. If 'completer' is present, it's used as the argcomplete completer for the argument. Args: args: Position args to pass to argparse add_argument *kwargs: Named args to pass to argparse add_argument """ completer = None if 'completer' in kwargs: completer = kwargs['completer'] del kwargs['completer'] self.args = args self.kwargs = kwargs self.completer = completer @staticmethod def MakeZeroOrMoreCloudURLsArgument(): """Constructs an argument that takes 0 or more Cloud URLs as parameters.""" return CommandArgument('file', nargs='', completer=CompleterType.CLOUD_OBJECT) @staticmethod def MakeZeroOrMoreCloudBucketURLsArgument(): """Constructs an argument that takes 0+ Cloud bucket URLs as parameters.""" return CommandArgument('file', nargs='', completer=CompleterType.CLOUD_BUCKET) @staticmethod def MakeNCloudBucketURLsArgument(n): """Constructs an argument that takes N Cloud bucket URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_BUCKET) @staticmethod def MakeNCloudURLsArgument(n): """Constructs an argument that takes N Cloud URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_OBJECT) @staticmethod def MakeZeroOrMoreCloudOrFileURLsArgument(): """Constructs an argument that takes 0 or more Cloud or File URLs.""" return CommandArgument('file', nargs='', completer=CompleterType.CLOUD_OR_LOCAL_OBJECT) @staticmethod def MakeNCloudOrFileURLsArgument(n): """Constructs an argument that takes N Cloud or File URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.CLOUD_OR_LOCAL_OBJECT) @staticmethod def MakeZeroOrMoreFileURLsArgument(): """Constructs an argument that takes 0 or more File URLs as parameters.""" return CommandArgument('file', nargs='*', completer=CompleterType.LOCAL_OBJECT) @staticmethod def MakeNFileURLsArgument(n): """Constructs an argument that takes N File URLs as parameters.""" return CommandArgument('file', nargs=n, completer=CompleterType.LOCAL_OBJECT) @staticmethod def MakeFileURLOrCannedACLArgument(): """Constructs an argument that takes a File URL or a canned ACL.""" return CommandArgument('file', nargs=1, completer=CompleterType.LOCAL_OBJECT_OR_CANNED_ACL) @staticmethod def MakeFreeTextArgument(): """Constructs an argument that takes arbitrary text.""" return CommandArgument('text', completer=CompleterType.NO_OP) @staticmethod def MakeOneOrMoreBindingsArgument(): """Constructs an argument that takes multiple bindings.""" return CommandArgument('binding', nargs='+', completer=CompleterType.NO_OP)	4.35 KB	2025-06-25 08:19
📄 command_runner.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Class that runs a named gsutil command.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import difflib import logging import os import pkgutil import sys import textwrap import time import six from six.moves import input import boto from boto import config from boto.storage_uri import BucketStorageUri import gslib from gslib import metrics from gslib.cloud_api_delegator import CloudApiDelegator from gslib.command import Command from gslib.command import CreateOrGetGsutilLogger from gslib.command import GetFailureCount from gslib.command import OLD_ALIAS_MAP from gslib.command import ShutDownGsutil import gslib.commands from gslib.cs_api_map import ApiSelector from gslib.cs_api_map import GsutilApiClassMapFactory from gslib.cs_api_map import GsutilApiMapFactory from gslib.discard_messages_queue import DiscardMessagesQueue from gslib.exception import CommandException from gslib.gcs_json_api import GcsJsonApi from gslib.no_op_credentials import NoOpCredentials from gslib.tab_complete import MakeCompleter from gslib.utils import boto_util from gslib.utils import shim_util from gslib.utils import system_util from gslib.utils.constants import RELEASE_NOTES_URL from gslib.utils.constants import UTF8 from gslib.utils.metadata_util import IsCustomMetadataHeader from gslib.utils.parallelism_framework_util import CheckMultiprocessingAvailableAndInit from gslib.utils.text_util import CompareVersions from gslib.utils.text_util import InsistAsciiHeader from gslib.utils.text_util import InsistAsciiHeaderValue from gslib.utils.text_util import print_to_fd from gslib.utils.unit_util import SECONDS_PER_DAY from gslib.utils.update_util import LookUpGsutilVersion from gslib.utils.update_util import GsutilPubTarball def HandleHeaderCoding(headers): """Handles coding of headers and their values. Alters the dict in-place. Converts a dict of headers and their values to their appropriate types. We ensure that all headers and their values will contain only ASCII characters, with the exception of custom metadata header values; these values may contain Unicode characters, and thus if they are not already unicode-type objects, we attempt to decode them to Unicode using UTF-8 encoding. Args: headers: A dict mapping headers to their values. All keys and values must be either str or unicode objects. Raises: CommandException: If a header or its value cannot be encoded in the required encoding. """ if not headers: return for key in headers: InsistAsciiHeader(key) if IsCustomMetadataHeader(key): if not isinstance(headers[key], six.text_type): try: headers[key] = headers[key].decode(UTF8) except UnicodeDecodeError: raise CommandException('\n'.join( textwrap.wrap( 'Invalid encoding for header value (%s: %s). Values must be ' 'decodable as Unicode. NOTE: the value printed above ' 'replaces the problematic characters with a hex-encoded ' 'printable representation. For more details (including how to ' 'convert to a gsutil-compatible encoding) see `gsutil help ' 'encoding`.' % (repr(key), repr(headers[key]))))) else: # Non-custom-metadata headers and their values must be ASCII characters. InsistAsciiHeaderValue(key, headers[key]) def HandleArgCoding(args): """Handles coding of command-line args. Alters the list in-place. Args: args: A list of command-line args. Raises: CommandException: if errors encountered. """ # Python passes arguments from the command line as byte strings. To # correctly interpret them, we decode them as utf-8. for i in range(len(args)): arg = args[i] if not isinstance(arg, six.text_type): try: args[i] = arg.decode(UTF8) except UnicodeDecodeError: raise CommandException('\n'.join( textwrap.wrap( 'Invalid encoding for argument (%s). Arguments must be decodable ' 'as Unicode. NOTE: the argument printed above replaces the ' 'problematic characters with a hex-encoded printable ' 'representation. For more details (including how to convert to a ' 'gsutil-compatible encoding) see `gsutil help encoding`.' % repr(arg)))) def _StringToSysArgType(unicode_str): """Converts a string literal (unicode) to the same type as sys.argv[0].""" # TODO(PY3-ONLY): If we remove the PY2 code branch, this method becomes # a no-op, so we can just remove the whole method when we move to PY3. if six.PY2: return unicode_str.encode(UTF8) return unicode_str class CommandRunner(object): """Runs gsutil commands and does some top-level argument handling.""" def __init__(self, bucket_storage_uri_class=BucketStorageUri, gsutil_api_class_map_factory=GsutilApiClassMapFactory, command_map=None): """Instantiates a CommandRunner. Args: bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. gsutil_api_class_map_factory: Creates map of cloud storage interfaces. Settable for testing/mocking. command_map: Map of command names to their implementations for testing/mocking. If not set, the map is built dynamically. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.gsutil_api_class_map_factory = gsutil_api_class_map_factory if command_map: self.command_map = command_map else: self.command_map = self._LoadCommandMap() def _LoadCommandMap(self): """Returns dict mapping each command_name to implementing class.""" # Import all gslib.commands submodules. for _, module_name, _ in pkgutil.iter_modules(gslib.commands.__path__): __import__('gslib.commands.%s' % module_name) command_map = {} # Only include Command subclasses in the dict. for command in Command.__subclasses__(): command_map[command.command_spec.command_name] = command for command_name_aliases in command.command_spec.command_name_aliases: command_map[command_name_aliases] = command return command_map def _GetTabCompleteLogger(self): """Returns a logger for tab completion.""" return CreateOrGetGsutilLogger('tab_complete') def _ConfigureCommandArgumentParserArguments(self, parser, subcommands_or_arguments, gsutil_api): """Creates parsers recursively for potentially nested subcommands. Args: parser: argparse parser object. subcommands_or_arguments: list of CommandArgument objects, or recursive dict with subcommand names as keys. gsutil_api: gsutil Cloud API instance to use. Raises: RuntimeError: if argument is configured with unsupported completer TypeError: if subcommands_or_arguments is not a dict or list """ logger = self._GetTabCompleteLogger() def HandleList(): for command_argument in subcommands_or_arguments: action = parser.add_argument(command_argument.args, command_argument.kwargs) if command_argument.completer: action.completer = MakeCompleter(command_argument.completer, gsutil_api) def HandleDict(): subparsers = parser.add_subparsers() for subcommand_name, subcommand_value in subcommands_or_arguments.items(): cur_subcommand_parser = subparsers.add_parser(subcommand_name, add_help=False) logger.info( 'Constructing argument parsers for {}'.format(subcommand_name)) self._ConfigureCommandArgumentParserArguments(cur_subcommand_parser, subcommand_value, gsutil_api) if isinstance(subcommands_or_arguments, list): HandleList() elif isinstance(subcommands_or_arguments, dict): HandleDict() else: error_format = ('subcommands_or_arguments {} should be list or dict, ' 'found type {}') raise TypeError( error_format.format(subcommands_or_arguments, type(subcommands_or_arguments))) def GetGsutilApiForTabComplete(self): """Builds and returns a gsutil_api based off gsutil_api_class_map_factory. Returns: the gsutil_api instance """ # This should match the support map for the "ls" command. support_map = { 'gs': [ApiSelector.XML, ApiSelector.JSON], 's3': [ApiSelector.XML] } default_map = {'gs': ApiSelector.JSON, 's3': ApiSelector.XML} gsutil_api_map = GsutilApiMapFactory.GetApiMap( self.gsutil_api_class_map_factory, support_map, default_map) gsutil_api = CloudApiDelegator(self.bucket_storage_uri_class, gsutil_api_map, self._GetTabCompleteLogger(), DiscardMessagesQueue(), debug=0) return gsutil_api def ConfigureCommandArgumentParsers(self, main_parser): """Configures argparse arguments and argcomplete completers for commands. Args: main_parser: argparse object that can be called to get subparsers to add subcommands (called just 'commands' in gsutil) """ gsutil_api = self.GetGsutilApiForTabComplete() # build a dict mapping from command name to the argparse arguments. # this dict has values with either a recursive dictionary or a list of # CommandArgument objects. command_to_argparse_arguments = { command.command_spec.command_name: command.command_spec.argparse_arguments for command in self.command_map.values() } # At this point command_to_argparse_arguments looks like # { # 'retention': { # 'set' : `set arguments array` # 'get' : `set arguments array` # ... # 'event': { # 'set': `event set arguments array` # 'release': `event release arguments array` # } # }, # ... other commands here # } # # Which will be passed into the helper and called recursively on the items # in the dict, with the base case being the arguments arrays, where the # arguments are added to the subparser for the lowest level command. self._ConfigureCommandArgumentParserArguments( main_parser, command_to_argparse_arguments, gsutil_api) def RunNamedCommand(self, command_name, args=None, headers=None, debug=0, trace_token=None, parallel_operations=False, skip_update_check=False, logging_filters=None, do_shutdown=True, perf_trace_token=None, user_project=None, collect_analytics=False): """Runs the named command. Used by gsutil main, commands built atop other commands, and tests. Args: command_name: The name of the command being run. args: Command-line args (arg0 = actual arg, not command name ala bash). headers: Dictionary containing optional HTTP headers to pass to boto. debug: Debug level to pass in to boto connection (range 0..3). trace_token: Trace token to pass to the underlying API. parallel_operations: Should command operations be executed in parallel? skip_update_check: Set to True to disable checking for gsutil updates. logging_filters: Optional list of logging.Filters to apply to this command's logger. do_shutdown: Stop all parallelism framework workers iff this is True. perf_trace_token: Performance measurement trace token to pass to the underlying API. user_project: The project to bill this request to. collect_analytics: Set to True to collect an analytics metric logging this command. Raises: CommandException: if errors encountered. Returns: Return value(s) from Command that was run. """ command_changed_to_update = False if (not skip_update_check and self.MaybeCheckForAndOfferSoftwareUpdate(command_name, debug)): command_name = 'update' command_changed_to_update = True args = [_StringToSysArgType('-n')] # Check for opt-in analytics. if system_util.IsRunningInteractively() and collect_analytics: metrics.CheckAndMaybePromptForAnalyticsEnabling() self.MaybePromptForPythonUpdate(command_name) if not args: args = [] # Include api_version header in all commands. api_version = boto.config.get_value('GSUtil', 'default_api_version', '1') if not headers: headers = {} headers['x-goog-api-version'] = api_version if command_name not in self.command_map: close_matches = difflib.get_close_matches(command_name, self.command_map.keys(), n=1) if close_matches: # Instead of suggesting a deprecated command alias, suggest the new # name for that command. translated_command_name = (OLD_ALIAS_MAP.get(close_matches[0], close_matches)[0]) print('Did you mean this?', file=sys.stderr) print('\t%s' % translated_command_name, file=sys.stderr) elif command_name == 'update' and gslib.IS_PACKAGE_INSTALL: sys.stderr.write( 'Update command is not supported for package installs; ' 'please instead update using your package manager.') raise CommandException('Invalid command "%s".' % command_name) # Call str() on this string because the type of objects in `args` differ # on Python 2 vs 3 (bytes vs unicode), and we want to compare using the # same as whatever is in `args`. if _StringToSysArgType('--help') in args: new_args = [command_name] original_command_class = self.command_map[command_name] subcommands = original_command_class.help_spec.subcommand_help_text.keys() for arg in args: if arg in subcommands: new_args.append(arg) break # Take the first match and throw away the rest. args = new_args command_name = 'help' HandleArgCoding(args) HandleHeaderCoding(headers) command_class = self.command_map[command_name] command_inst = command_class(self, args, headers, debug, trace_token, parallel_operations, self.bucket_storage_uri_class, self.gsutil_api_class_map_factory, logging_filters, command_alias_used=command_name, perf_trace_token=perf_trace_token, user_project=user_project) # Log the command name, command alias, and sub-options after being parsed by # RunCommand and the command constructor. For commands with subcommands and # suboptions, we need to log the suboptions again within the command itself # because the command constructor will not parse the suboptions fully. if collect_analytics: metrics.LogCommandParams(command_name=command_inst.command_name, sub_opts=command_inst.sub_opts, command_alias=command_name) if command_inst.translate_to_gcloud_storage_if_requested(): # This does not mean that the gcloud storage command worked. # It only means that we succesfully attempted running gcloud storage. # The command itself might have failed. return_code = command_inst.run_gcloud_storage() else: # Run gsutil. return_code = command_inst.RunCommand() if CheckMultiprocessingAvailableAndInit().is_available and do_shutdown: ShutDownGsutil() if GetFailureCount() > 0: return_code = 1 if command_changed_to_update: # If the command changed to update, the user's original command was # not executed. return_code = 1 print('\n'.join( textwrap.wrap( 'Update was successful. Exiting with code 1 as the original command ' 'issued prior to the update was not executed and should be re-run.' ))) return return_code def SkipUpdateCheck(self): """Helper function that will determine if update checks should be skipped. Args: command_name: The name of the command being run. Returns: True if: - gsutil is not connected to a tty (e.g., if being run from cron); - user is running gsutil -q - user specified gs_host (which could be a non-production different service instance, in which case credentials won't work for checking gsutil tarball).""" logger = logging.getLogger() if (not system_util.IsRunningInteractively() or not logger.isEnabledFor(logging.INFO) or boto_util.HasUserSpecifiedGsHost()): return True return False def MaybePromptForPythonUpdate(self, command_name): """Alert the user that they should install Python 3. Args: command_name: The name of the command being run. Returns: True if a prompt was output. """ if (sys.version_info.major != 2 or self.SkipUpdateCheck() or command_name not in ('update', 'ver', 'version') or boto.config.getbool('GSUtil', 'skip_python_update_prompt', False)): return False # Notify the user about Python 2 deprecation. print_to_fd( 'Gsutil 5 drops Python 2 support. Please install Python 3 to update ' 'to the latest version of gsutil. https://goo.gle/py3\n') return True def MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug): """Checks the last time we checked for an update and offers one if needed. Offer is made if the time since the last update check is longer than the configured threshold offers the user to update gsutil. Args: command_name: The name of the command being run. debug: Debug level to pass in to boto connection (range 0..3). Returns: True if the user decides to update. """ # Don't try to interact with user if: # - SkipUpdateChecks returns True. # - user is running the config command (which could otherwise attempt to # check for an update for a user running behind a proxy, who has not yet # configured gsutil to go through the proxy; for such users we need the # first connection attempt to be made by the gsutil config command). # - user is running the version command (which gets run when using # gsutil -D, which would prevent users with proxy config problems from # sending us gsutil -D output). # - user is running the update command (which could otherwise cause an # additional note that an update is available when user is already trying # to perform an update); # - user is using a Cloud SDK install (which should only be updated via # gcloud components update) logger = logging.getLogger() if (self.SkipUpdateCheck() or command_name in ('config', 'update', 'ver', 'version') or system_util.InvokedViaCloudSdk()): return False software_update_check_period = boto.config.getint( 'GSUtil', 'software_update_check_period', 30) # Setting software_update_check_period to 0 means periodic software # update checking is disabled. if software_update_check_period == 0: return False last_checked_for_gsutil_update_timestamp_file = ( boto_util.GetLastCheckedForGsutilUpdateTimestampFile()) cur_ts = int(time.time()) if not os.path.isfile(last_checked_for_gsutil_update_timestamp_file): # Set last_checked_ts from date of VERSION file, so if the user installed # an old copy of gsutil it will get noticed (and an update offered) the # first time they try to run it. last_checked_ts = gslib.GetGsutilVersionModifiedTime() with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f: f.write(str(last_checked_ts)) else: try: with open(last_checked_for_gsutil_update_timestamp_file, 'r') as f: last_checked_ts = int(f.readline()) except (TypeError, ValueError): return False if (cur_ts - last_checked_ts > software_update_check_period SECONDS_PER_DAY): # Create a credential-less gsutil API to check for the public # update tarball. gsutil_api = GcsJsonApi(self.bucket_storage_uri_class, logger, DiscardMessagesQueue(), credentials=NoOpCredentials(), debug=debug) cur_ver = gslib.VERSION try: cur_ver = LookUpGsutilVersion(gsutil_api, GsutilPubTarball()) except Exception: return False with open(last_checked_for_gsutil_update_timestamp_file, 'w') as f: f.write(str(cur_ts)) (g, m) = CompareVersions(cur_ver, gslib.VERSION) if m: print_to_fd('\n'.join( textwrap.wrap( 'A newer version of gsutil (%s) is available than the version you ' 'are running (%s). NOTE: This is a major new version, so it is ' 'strongly recommended that you review the release note details at ' '%s before updating to this version, especially if you use gsutil ' 'in scripts.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))) if gslib.IS_PACKAGE_INSTALL: return False print_to_fd('\n') answer = input('Would you like to update [y/N]? ') return answer and answer.lower()[0] == 'y' elif g: print_to_fd('\n'.join( textwrap.wrap( 'A newer version of gsutil (%s) is available than the version you ' 'are running (%s). A detailed log of gsutil release changes is ' 'available at %s if you would like to read them before updating.' % (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))) if gslib.IS_PACKAGE_INSTALL: return False print_to_fd('\n') answer = input('Would you like to update [Y/n]? ') return not answer or answer.lower()[0] != 'n' return False	23.21 KB	2025-06-25 08:19
📁 commands	-	2025-06-25 08:26
📄 context_config.py # -- coding: utf-8 -- # Copyright 2020 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Manages device context mTLS certificates.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import atexit import json import os from boto import config import gslib from gslib import exception from gslib.utils import boto_util from gslib.utils import execution_util # Maintain a single context configuration. _singleton_config = None # Metadata JSON that stores information about the default certificate provider. _DEFAULT_METADATA_PATH = os.path.expanduser( os.path.join('~', '.secureConnect', 'context_aware_metadata.json')) _CERT_PROVIDER_COMMAND = "cert_provider_command" _CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION = "--with_passphrase" class CertProvisionError(Exception): """Represents errors when provisioning a client certificate.""" pass class ContextConfigSingletonAlreadyExistsError(Exception): """Error for when create_context_config is called multiple times.""" pass def _is_pem_section_marker(line): """Returns (begin:bool, end:bool, name:str).""" if line.startswith('-----BEGIN ') and line.endswith('-----'): return True, False, line[11:-5] elif line.startswith('-----END ') and line.endswith('-----'): return False, True, line[9:-5] else: return False, False, '' def _split_pem_into_sections(contents, logger): """Returns dict with {name: section} by parsing contents in PEM format. A simple parser for PEM file. Please see RFC 7468 for the format of PEM file. Not using regex to improve performance catching nested matches. Note: This parser requires the post-encapsulation label of a section to match its pre-encapsulation label. It ignores a section without a matching label. Args: contents (str): Contents of a PEM file. logger (logging.logger): gsutil logger. Returns: A dict of the PEM file sections. """ result = {} pem_lines = [] pem_section_name = None for line in contents.splitlines(): line = line.strip() if not line: continue begin, end, name = _is_pem_section_marker(line) if begin: if pem_section_name: logger.warning('Section %s missing end line and will be ignored.' % pem_section_name) if name in result.keys(): logger.warning('Section %s already exists, and the older section will ' 'be ignored.' % name) pem_section_name = name pem_lines = [] elif end: if not pem_section_name: logger.warning( 'Section %s missing a beginning line and will be ignored.' % name) elif pem_section_name != name: logger.warning('Section %s missing a matching end line. Found: %s' % (pem_section_name, name)) pem_section_name = None if pem_section_name: pem_lines.append(line) if end: result[name] = '\n'.join(pem_lines) + '\n' pem_section_name = None if pem_section_name: logger.warning('Section %s missing an end line.' % pem_section_name) return result def _check_path(): """Checks for content aware metadata. If content aware metadata exists, return its absolute path; otherwise, returns None. Returns: str: Absolute path if exists. Otherwise, None. """ metadata_path = os.path.expanduser(_DEFAULT_METADATA_PATH) if not os.path.exists(metadata_path): return None return metadata_path def _read_metadata_file(metadata_path): """Loads context aware metadata from the given path. Returns: dict: The metadata JSON. Raises: CertProvisionError: If failed to parse metadata as JSON. """ try: with open(metadata_path) as f: return json.load(f) except ValueError as e: raise CertProvisionError(e) def _default_command(): """Loads default cert provider command. Returns: str: The default command. Raises: CertProvisionError: If command cannot be found. """ metadata_path = _check_path() if not metadata_path: raise CertProvisionError("Client certificate provider file not found.") metadata_json = _read_metadata_file(metadata_path) if _CERT_PROVIDER_COMMAND not in metadata_json: raise CertProvisionError("Client certificate provider command not found.") command = metadata_json[_CERT_PROVIDER_COMMAND] if (_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION not in command): command.append(_CERT_PROVIDER_COMMAND_PASSPHRASE_OPTION) return command class _ContextConfig(object): """Represents the configurations associated with context aware access. Only one instance of Config can be created for the program. """ def __init__(self, logger): """Initializes config. Args: logger (logging.logger): gsutil logger. """ self.logger = logger self.use_client_certificate = config.getbool('Credentials', 'use_client_certificate') self.client_cert_path = None if not self.use_client_certificate: # Don't spend time generating values gsutil won't need. return # Generates certificate and deletes it afterwards. atexit.register(self._unprovision_client_cert) self.client_cert_path = os.path.join(boto_util.GetGsutilStateDir(), 'caa_cert.pem') try: # Certs provisioned using endpoint verification are stored as a # single file holding both the public certificate and the private key. self._provision_client_cert(self.client_cert_path) except CertProvisionError as e: self.logger.error('Failed to provision client certificate: %s' % e) def _provision_client_cert(self, cert_path): """Executes certificate provider to obtain client certificate and keys.""" cert_command_string = config.get('Credentials', 'cert_provider_command', None) if cert_command_string: cert_command = cert_command_string.split(' ') else: # Load the default certificate provider if sit is not provided by user. cert_command = _default_command() try: command_stdout_string, _ = execution_util.ExecuteExternalCommand( cert_command) sections = _split_pem_into_sections(command_stdout_string, self.logger) with open(cert_path, 'w+') as f: f.write(sections['CERTIFICATE']) if 'ENCRYPTED PRIVATE KEY' in sections: f.write(sections['ENCRYPTED PRIVATE KEY']) self.client_cert_password = sections['PASSPHRASE'].splitlines()[1] else: f.write(sections['PRIVATE KEY']) self.client_cert_password = None except (exception.ExternalBinaryError, OSError) as e: raise CertProvisionError(e) except KeyError as e: raise CertProvisionError( 'Invalid output format from certificate provider, no %s' % e) def _unprovision_client_cert(self): """Cleans up any files or resources provisioned during config init.""" if self.client_cert_path is not None: try: os.remove(self.client_cert_path) self.logger.debug('Unprovisioned client cert: %s' % self.client_cert_path) except OSError as e: self.logger.error('Failed to remove client certificate: %s' % e) def create_context_config(logger): """Should be run once at gsutil startup. Creates global singleton. Args: logger (logging.logger): For logging during config functions. Returns: New ContextConfig singleton. Raises: Exception if singleton already exists. """ global _singleton_config if not _singleton_config: _singleton_config = _ContextConfig(logger) return _singleton_config raise ContextConfigSingletonAlreadyExistsError def get_context_config(): """Retrieves ContextConfig global singleton. Returns: ContextConfig or None if global singleton doesn't exist. """ return _singleton_config	8.36 KB	2025-06-25 08:19
📄 cred_types.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Defines gsutil-supported credential types.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class CredTypes(object): HMAC = "HMAC" IMPERSONATION = "Service Account Impersonation" OAUTH2_SERVICE_ACCOUNT = "OAuth 2.0 Service Account" OAUTH2_USER_ACCOUNT = "Oauth 2.0 User Account" GCE = "GCE" DEVSHELL = "Google Developer Shell" EXTERNAL_ACCOUNT = "External Account" EXTERNAL_ACCOUNT_AUTHORIZED_USER = "External Account Authorized User"	1.15 KB	2025-06-25 08:19
📄 cs_api_map.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """API map classes used with the CloudApiDelegator class.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from gslib.boto_translation import BotoTranslation from gslib.gcs_json_api import GcsJsonApi class ApiSelector(object): """Enum class for API.""" XML = 'XML' JSON = 'JSON' class ApiMapConstants(object): """Enum class for API map entries.""" API_MAP = 'apiclass' SUPPORT_MAP = 'supported' DEFAULT_MAP = 'default' class GsutilApiClassMapFactory(object): """Factory for generating gsutil API class maps. A valid class map is defined as: { (key) Provider prefix used in URI strings. (value) { (key) ApiSelector describing the API format. (value) CloudApi child class that implements this API. } } """ @classmethod def GetClassMap(cls): """Returns the default gsutil class map.""" gs_class_map = { ApiSelector.XML: BotoTranslation, ApiSelector.JSON: GcsJsonApi } s3_class_map = { ApiSelector.XML: BotoTranslation, } class_map = { 'gs': gs_class_map, 's3': s3_class_map, } return class_map class GsutilApiMapFactory(object): """Factory the generates the default gsutil API map. The API map determines which Cloud API implementation is used for a given command. A valid API map is defined as: { (key) ApiMapConstants.API_MAP : (value) Gsutil API class map (as described in GsutilApiClassMapFactory comments). (key) ApiMapConstants.SUPPORT_MAP : (value) { (key) Provider prefix used in URI strings. (value) list of ApiSelectors supported by the command for this provider. } (key) ApiMapConstants.DEFAULT_MAP : (value) { (key) Provider prefix used in URI strings. (value) Default ApiSelector for this command and provider. } } """ @classmethod def GetApiMap(cls, gsutil_api_class_map_factory, support_map, default_map): """Creates a GsutilApiMap for use by the command from the inputs. Args: gsutil_api_class_map_factory: Factory defining a GetClassMap() function adhering to GsutilApiClassMapFactory semantics. support_map: Entries for ApiMapConstants.SUPPORT_MAP as described above. default_map: Entries for ApiMapConstants.DEFAULT_MAP as described above. Returns: GsutilApiMap generated from the inputs. """ return { ApiMapConstants.API_MAP: gsutil_api_class_map_factory.GetClassMap(), ApiMapConstants.SUPPORT_MAP: support_map, ApiMapConstants.DEFAULT_MAP: default_map }	3.31 KB	2025-06-25 08:19
📄 daisy_chain_wrapper.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Wrapper for use in daisy-chained copies.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import collections import contextlib import os import threading import time from gslib.cloud_api import BadRequestException from gslib.cloud_api import CloudApi from gslib.utils import constants from gslib.utils import parallelism_framework_util from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey # This controls the amount of bytes downloaded per download request. # We do not buffer this many bytes in memory at a time - that is controlled by # DaisyChainWrapper.max_buffer_size. This is the upper bound of bytes that may # be unnecessarily downloaded if there is a break in the resumable upload. _DEFAULT_DOWNLOAD_CHUNK_SIZE = 1024 * 1024 * 100 class BufferWrapper(object): """Wraps the download file pointer to use our in-memory buffer.""" def __init__(self, daisy_chain_wrapper, mode='b'): """Provides a buffered write interface for a file download. Args: daisy_chain_wrapper: DaisyChainWrapper instance to use for buffer and locking. """ self.daisy_chain_wrapper = daisy_chain_wrapper if hasattr(daisy_chain_wrapper, 'mode'): self.mode = daisy_chain_wrapper.mode else: self.mode = mode def write(self, data): # pylint: disable=invalid-name """Waits for space in the buffer, then writes data to the buffer.""" while True: with self.daisy_chain_wrapper.lock: if (self.daisy_chain_wrapper.bytes_buffered < self.daisy_chain_wrapper.max_buffer_size): break # Buffer was full, yield thread priority so the upload can pull from it. time.sleep(0) data_len = len(data) if data_len: with self.daisy_chain_wrapper.lock: self.daisy_chain_wrapper.buffer.append(data) self.daisy_chain_wrapper.bytes_buffered += data_len @contextlib.contextmanager def AcquireLockWithTimeout(lock, timeout): # pylint: disable=invalid-name result = lock.acquire(timeout=timeout) yield result if result: lock.release() class DaisyChainWrapper(object): """Wrapper class for daisy-chaining a cloud download to an upload. This class instantiates a BufferWrapper object to buffer the download into memory, consuming a maximum of max_buffer_size. It implements intelligent behavior around read and seek that allow for all of the operations necessary to copy a file. This class is coupled with the XML and JSON implementations in that it expects that small buffers (maximum of constants.TRANSFER_BUFFER_SIZE) in size will be used. """ def __init__(self, src_url, src_obj_size, gsutil_api, compressed_encoding=False, progress_callback=None, download_chunk_size=_DEFAULT_DOWNLOAD_CHUNK_SIZE, decryption_key=None): """Initializes the daisy chain wrapper. Args: src_url: Source CloudUrl to copy from. src_obj_size: Size of source object. gsutil_api: gsutil Cloud API to use for the copy. compressed_encoding: If true, source object has content-encoding: gzip. progress_callback: Optional callback function for progress notifications for the download thread. Receives calls with arguments (bytes_transferred, total_size). download_chunk_size: Integer number of bytes to download per GetObjectMedia request. This is the upper bound of bytes that may be unnecessarily downloaded if there is a break in the resumable upload. decryption_key: Base64-encoded decryption key for the source object, if any. Raises: Exception: if the download thread doesn't start within 60 seconds """ # Current read position for the upload file pointer. self.position = 0 self.buffer = collections.deque() self.bytes_buffered = 0 # Maximum amount of bytes in memory at a time. self.max_buffer_size = 1024 * 1024 # 1 MiB self._download_chunk_size = download_chunk_size # We save one buffer's worth of data as a special case for boto, # which seeks back one buffer and rereads to compute hashes. This is # unnecessary because we can just compare cloud hash digests at the end, # but it allows this to work without modfiying boto. self.last_position = 0 self.last_data = None # Protects buffer, position, bytes_buffered, last_position, and last_data. self.lock = parallelism_framework_util.CreateLock() # Protects download_exception. self.download_exception_lock = parallelism_framework_util.CreateLock() self.src_obj_size = src_obj_size self.src_url = src_url self.compressed_encoding = compressed_encoding self.decryption_tuple = CryptoKeyWrapperFromKey(decryption_key) # This is safe to use the upload and download thread because the download # thread calls only GetObjectMedia, which creates a new HTTP connection # independent of gsutil_api. Thus, it will not share an HTTP connection # with the upload. self.gsutil_api = gsutil_api # If self.download_thread dies due to an exception, it is saved here so # that it can also be raised in the upload thread. self.download_exception = None self.download_thread = None self.progress_callback = progress_callback self.download_started = threading.Event() self.stop_download = threading.Event() self.StartDownloadThread(progress_callback=self.progress_callback) if not self.download_started.wait(60): raise Exception('Could not start download thread after 60 seconds.') self._unused_data_from_previous_read = b'' def StartDownloadThread(self, start_byte=0, progress_callback=None): # pylint: disable=invalid-name """Starts the download thread for the source object (from start_byte).""" def PerformDownload(start_byte, progress_callback): # pylint: disable=invalid-name """Downloads the source object in chunks. This function checks the stop_download event and exits early if it is set. It should be set when there is an error during the daisy-chain upload, then this function can be called again with the upload's current position as start_byte. Args: start_byte: Byte from which to begin the download. progress_callback: Optional callback function for progress notifications. Receives calls with arguments (bytes_transferred, total_size). """ # TODO: Support resumable downloads. This would require the BufferWrapper # object to support seek() and tell() which requires coordination with # the upload. self.download_started.set() try: while start_byte + self._download_chunk_size < self.src_obj_size: self.gsutil_api.GetObjectMedia( self.src_url.bucket_name, self.src_url.object_name, BufferWrapper(self), compressed_encoding=self.compressed_encoding, start_byte=start_byte, end_byte=start_byte + self._download_chunk_size - 1, generation=self.src_url.generation, object_size=self.src_obj_size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=self.src_url.scheme, progress_callback=progress_callback, decryption_tuple=self.decryption_tuple) if self.stop_download.is_set(): # Download thread needs to be restarted, so exit. self.stop_download.clear() return start_byte += self._download_chunk_size self.gsutil_api.GetObjectMedia( self.src_url.bucket_name, self.src_url.object_name, BufferWrapper(self), compressed_encoding=self.compressed_encoding, start_byte=start_byte, generation=self.src_url.generation, object_size=self.src_obj_size, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, provider=self.src_url.scheme, progress_callback=progress_callback, decryption_tuple=self.decryption_tuple) # We catch all exceptions here because we want to store them. except Exception as e: # pylint: disable=broad-except # Save the exception so that it can be seen in the upload thread. with self.download_exception_lock: self.download_exception = e raise # TODO: If we do gzip encoding transforms mid-transfer, this will fail. self.download_thread = threading.Thread(target=PerformDownload, args=(start_byte, progress_callback)) self.download_thread.start() def read(self, amt=None): # pylint: disable=invalid-name """Exposes a stream from the in-memory buffer to the upload.""" if self.position == self.src_obj_size or amt == 0: # If there is no data left or 0 bytes were requested, return an empty # string so callers can call still call len() and read(0). return '' if amt is None or amt > constants.TRANSFER_BUFFER_SIZE: raise BadRequestException( 'Invalid HTTP read size %s during daisy chain operation, ' 'expected <= %s.' % (amt, constants.TRANSFER_BUFFER_SIZE)) if self._unused_data_from_previous_read: valid_data = self._unused_data_from_previous_read[:amt] self._unused_data_from_previous_read = ( self._unused_data_from_previous_read[amt:]) else: valid_data = b'' while True: with self.lock: if self.buffer: break if AcquireLockWithTimeout(self.download_exception_lock, 30): if self.download_exception: # Download thread died, so we will never recover. Raise the # exception that killed it. raise self.download_exception # pylint: disable=raising-bad-type else: if not self.download_thread.is_alive(): raise Exception('Download thread died suddenly.') # Buffer was empty, yield thread priority so the download thread can fill. time.sleep(0) with self.lock: # In a rare case where len(valid_data) < amt, and there are # elements in the buffer, e.g let's say valid_data = 'foo' and amt = 5, # and we have 'barbaz' in self.buffer, instead of returning 'fooba' # in the current call, we return 'foo', and in the subsequent calls # we will return 'barba' and 'z'. Given that we set # TRANSFER_BUFFER_SIZE as a multiple of the amt (8KiB), # we can safely assume that this will never happen practically. # This assumption helps us keep the logic simple and avoid # string concatenations and splits ('foo' + 'ba' in the above case). if not valid_data: data = self.buffer.popleft() valid_data = data[:amt] self._unused_data_from_previous_read = data[amt:] self.last_position = self.position self.last_data = valid_data data_len = len(valid_data) self.position += data_len self.bytes_buffered -= data_len if data_len > amt: raise BadRequestException( 'Invalid read during daisy chain operation, got data of size ' '%s, expected size %s.' % (data_len, amt)) return valid_data def tell(self): # pylint: disable=invalid-name with self.lock: return self.position def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Sets current read position of the stream.""" restart_download = False if whence == os.SEEK_END: if offset: raise IOError( 'Invalid seek during daisy chain operation. Non-zero offset %s ' 'from os.SEEK_END is not supported' % offset) with self.lock: self.last_position = self.position self.last_data = None # Safe because we check position against src_obj_size in read. self.position = self.src_obj_size elif whence == os.SEEK_SET: with self.lock: if offset == self.position: pass elif offset == self.last_position: self.position = self.last_position if self.last_data: # If we seek to end and then back, we won't have last_data; we'll # get it on the next call to read. self.buffer.appendleft(self.last_data) self.bytes_buffered += len(self.last_data) else: # Once a download is complete, boto seeks to 0 and re-reads to # compute the hash if an md5 isn't already present (for example a GCS # composite object), so we have to re-download the whole object. # Also, when daisy-chaining to a resumable upload, on error the # service may have received any number of the bytes; the download # needs to be restarted from that point. restart_download = True if restart_download: self.stop_download.set() # Consume any remaining bytes in the download thread so that # the thread can exit, then restart the thread at the desired position. while self.download_thread.is_alive(): with self.lock: while self.bytes_buffered: self.bytes_buffered -= len(self.buffer.popleft()) time.sleep(0) with self.lock: self.position = offset self.buffer = collections.deque() self.bytes_buffered = 0 self.last_position = 0 self.last_data = None self.stop_download.clear() self.StartDownloadThread(start_byte=offset, progress_callback=self.progress_callback) else: raise IOError('Daisy-chain download wrapper does not support ' 'seek mode %s' % whence) def seekable(self): # pylint: disable=invalid-name return True	14.39 KB	2025-06-25 08:19
📁 data	-	2025-06-25 08:19
📄 devshell_auth_plugin.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Developer Shell auth bridge. This enables Boto API auth in Developer Shell environment. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto.auth_handler import AuthHandler from boto.auth_handler import NotReadyToAuthenticate import oauth2client.contrib.devshell as devshell class DevshellAuth(AuthHandler): """Developer Shell authorization plugin class.""" capability = ['s3'] def __init__(self, path, config, provider): # Provider here is a boto.provider.Provider object (as opposed to the # provider attribute of CloudApi objects, which is a string). if provider.name != 'google': # Devshell credentials are valid for Google only and can't be used for s3. raise NotReadyToAuthenticate() try: self.creds = devshell.DevshellCredentials() except: raise NotReadyToAuthenticate() def add_auth(self, http_request): http_request.headers['Authorization'] = ('Bearer %s' % self.creds.access_token)	1.69 KB	2025-06-25 08:19
📄 discard_messages_queue.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a message queue that discards all messages.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class DiscardMessagesQueue(object): """Emulates a Cloud API status queue but drops all messages. This is useful when you want to perform some operations but not have the UI thread display information about those ops (e.g. running a test or fetching the public gsutil tarball object's metadata to perform a version check). """ # pylint: disable=invalid-name, unused-argument def put(self, message=None, timeout=None): pass # pylint: enable=invalid-name, unused-argument	1.29 KB	2025-06-25 08:19
📄 exception.py # -- coding: utf-8 -- # Copyright 2010 Google Inc. All Rights Reserved. # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, dis- # tribute, sublicense, and/or sell copies of the Software, and to permit # persons to whom the Software is furnished to do so, subject to the fol- # lowing conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. """gsutil exceptions. The exceptions in this module are for use across multiple different classes. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import six NO_URLS_MATCHED_PREFIX = 'No URLs matched' NO_URLS_MATCHED_GENERIC = (NO_URLS_MATCHED_PREFIX + '. Do the files you\'re operating on exist?') NO_URLS_MATCHED_TARGET = NO_URLS_MATCHED_PREFIX + ': %s' if six.PY3: # StandardError was removed, so use the base exception type instead StandardError = Exception class AbortException(StandardError): """Exception raised when a user aborts a command that needs to do cleanup.""" def __init__(self, reason): StandardError.__init__(self) self.reason = reason def __repr__(self): return 'AbortException: %s' % self.reason def __str__(self): return 'AbortException: %s' % self.reason class CommandException(StandardError): """Exception raised when a problem is encountered running a gsutil command. This exception should be used to signal user errors or system failures (like timeouts), not bugs (like an incorrect param value). For the latter you should raise Exception so we can see where/how it happened via gsutil -D (which will include a stack trace for raised Exceptions). """ def __init__(self, reason, informational=False): """Instantiate a CommandException. Args: reason: Text describing the problem. informational: Indicates reason should be printed as FYI, not a failure. """ StandardError.__init__(self) self.reason = reason self.informational = informational def __repr__(self): return str(self) def __str__(self): return 'CommandException: %s' % self.reason class ControlCException(Exception): """Exception to report to analytics when the user exits via ctrl-C. This exception is never actually raised, but is used by analytics collection to provide a more descriptive name for user exit. """ pass class GcloudStorageTranslationError(Exception): """Exception raised when a gsutil command can't be translated to gcloud.""" pass class HashMismatchException(Exception): """Exception raised when data integrity validation fails.""" pass class IamChOnResourceWithConditionsException(Exception): """Raised when trying to use "iam ch" on an IAM policy with conditions. Because the syntax for conditions is fairly complex, it doesn't make sense to specify them on the command line using a colon-delimited set of values in the same way you'd specify simple bindings - it would be a complex and potentially surprising interface, which isn't what you want when dealing with permissions. Additionally, providing partial functionality -- e.g. if a policy contains bindings with conditions, still allow users to interact with bindings that don't contain conditions -- might sound tempting, but results in a bad user experience. Bindings can be thought of as a mapping from (role, condition) -> [members]. Thus, a user might think they're editing the binding for (role1, condition1), but they'd really be editing the binding for (role1, None). Thus, we just raise an error if we encounter a binding with conditions present, and encourage users to use "iam {get,set}" instead. """ def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return str(self) def __str__(self): return 'IamChOnResourceWithConditionsException: %s' % self.message class InvalidUrlError(Exception): """Exception raised when URL is invalid.""" def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return str(self) def __str__(self): return 'InvalidUrlError: %s' % self.message class ExternalBinaryError(Exception): """Exception raised when gsutil runs an external binary, and it fails.""" def __init__(self, message): Exception.__init__(self, message) self.message = message def __repr__(self): return 'ExternalBinaryError: %s' % self.message	5.18 KB	2025-06-25 08:19
📄 file_part.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """FilePart implementation for representing part of a file.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import io class FilePart(io.IOBase): """Subclass of the file API for representing part of a file. This class behaves as a contiguous subset of a given file (e.g., this object will behave as though the desired part of the file was written to another file, and the second file was opened). """ # pylint: disable=super-init-not-called def __init__(self, filename, offset, length): """Initializes the FilePart. Args: filename: The name of the existing file, of which this object represents a part. offset: The position (in bytes) in the original file that corresponds to the first byte of the FilePart. length: The total number of bytes in the FilePart. """ self._fp = open(filename, 'rb') self.length = length self._start = offset self._end = self._start + self.length self._fp.seek(self._start) def __enter__(self): pass # pylint: disable=redefined-builtin def __exit__(self, type, value, traceback): self.close() def tell(self): return self._fp.tell() - self._start def read(self, size=-1): if size < 0: size = self.length size = min(size, self._end - self._fp.tell()) # Only read to our EOF return self._fp.read(max(0, size)) def seek(self, offset, whence=os.SEEK_SET): if whence == os.SEEK_END: return self._fp.seek(offset + self._end) elif whence == os.SEEK_CUR: return self._fp.seek(offset, whence) else: return self._fp.seek(self._start + offset) def close(self): self._fp.close() def flush(self, size=None): raise NotImplementedError('flush is not implemented in FilePart.') def fileno(self, size=None): raise NotImplementedError('fileno is not implemented in FilePart.') def isatty(self, size=None): raise NotImplementedError('isatty is not implemented in FilePart.') def next(self, size=None): raise NotImplementedError('next is not implemented in FilePart.') def readline(self, size=None): raise NotImplementedError('readline is not implemented in FilePart.') def readlines(self, size=None): raise NotImplementedError('readlines is not implemented in FilePart.') def xreadlines(self, size=None): raise NotImplementedError('xreadlines is not implemented in FilePart.') def truncate(self, size=None): raise NotImplementedError('truncate is not implemented in FilePart.') def write(self, size=None): raise NotImplementedError('write is not implemented in FilePart.') def writelines(self, size=None): raise NotImplementedError('writelines is not implemented in FilePart.')	3.39 KB	2025-06-25 08:19
📄 gcs_json_api.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from contextlib import contextmanager import functools from six.moves import http_client import json import logging import os import socket import ssl import time import traceback import six from apitools.base.py import encoding from apitools.base.py import exceptions as apitools_exceptions from apitools.base.py import http_wrapper as apitools_http_wrapper from apitools.base.py import transfer as apitools_transfer from apitools.base.py.util import CalculateWaitForRetry from boto import config import httplib2 import oauth2client from gslib import context_config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import ArgumentException from gslib.cloud_api import BadRequestException from gslib.cloud_api import CloudApi from gslib.cloud_api import EncryptionException from gslib.cloud_api import NotEmptyException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import Preconditions from gslib.cloud_api import PublishPermissionDeniedException from gslib.cloud_api import ResumableDownloadException from gslib.cloud_api import ResumableUploadAbortException from gslib.cloud_api import ResumableUploadException from gslib.cloud_api import ResumableUploadStartOverException from gslib.cloud_api import ServiceException from gslib.exception import CommandException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.gcs_json_credentials import isP12Credentials from gslib.gcs_json_media import BytesTransferredContainer from gslib.gcs_json_media import DownloadCallbackConnectionClassFactory from gslib.gcs_json_media import HttpWithDownloadStream from gslib.gcs_json_media import HttpWithNoRetries from gslib.gcs_json_media import UploadCallbackConnectionClassFactory from gslib.gcs_json_media import WrapDownloadHttpRequest from gslib.gcs_json_media import WrapUploadHttpRequest from gslib.impersonation_credentials import ImpersonationCredentials from gslib.no_op_credentials import NoOpCredentials from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.project_id import PopulateProjectId from gslib.third_party.storage_apitools import storage_v1_client as apitools_client from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.tracker_file import DeleteTrackerFile from gslib.tracker_file import GetRewriteTrackerFilePath from gslib.tracker_file import HashRewriteParameters from gslib.tracker_file import ReadRewriteTrackerFile from gslib.tracker_file import WriteRewriteTrackerFile from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetGcsJsonApiVersion from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries from gslib.utils.boto_util import JsonResumableChunkSizeDefined from gslib.utils.cloud_api_helper import ListToGetFields from gslib.utils.cloud_api_helper import ValidateDstObjectMetadata from gslib.utils.constants import IAM_POLICY_VERSION from gslib.utils.constants import NUM_OBJECTS_PER_LIST_PAGE from gslib.utils.constants import REQUEST_REASON_ENV_VAR from gslib.utils.constants import REQUEST_REASON_HEADER_KEY from gslib.utils.constants import UTF8 from gslib.utils.encryption_helper import Base64Sha256FromBase64EncryptionKey from gslib.utils.encryption_helper import CryptoKeyType from gslib.utils.encryption_helper import CryptoKeyWrapperFromKey from gslib.utils.encryption_helper import FindMatchingCSEKInBotoConfig from gslib.utils.metadata_util import AddAcceptEncodingGzipIfNeeded from gslib.utils.retry_util import LogAndHandleRetries from gslib.utils.signurl_helper import CreatePayload, GetFinalUrl from gslib.utils.text_util import GetPrintableExceptionString from gslib.utils.translation_helper import CreateBucketNotFoundException from gslib.utils.translation_helper import CreateNotFoundExceptionForObjectWrite from gslib.utils.translation_helper import CreateObjectNotFoundException from gslib.utils.translation_helper import DEFAULT_CONTENT_TYPE from gslib.utils.translation_helper import PRIVATE_DEFAULT_OBJ_ACL from gslib.utils.translation_helper import REMOVE_CORS_CONFIG from oauth2client.service_account import ServiceAccountCredentials if six.PY3: long = int # pylint: disable=invalid-name Notification = apitools_messages.Notification NotificationCustomAttributesValue = Notification.CustomAttributesValue NotificationAdditionalProperty = ( NotificationCustomAttributesValue.AdditionalProperty) # Implementation supports only 'gs' URLs, so provider is unused. # pylint: disable=unused-argument NUM_BUCKETS_PER_LIST_PAGE = 1000 TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError, apitools_exceptions.StreamExhausted, apitools_exceptions.TransferError, apitools_exceptions.TransferInvalidError) # TODO: Distribute these exceptions better through apitools and here. # Right now, apitools is configured not to handle any exceptions on # uploads/downloads. # oauth2_client tries to JSON-decode the response, which can result # in a ValueError if the response was invalid. Until that is fixed in # oauth2_client, need to handle it here. HTTP_TRANSFER_EXCEPTIONS = ( apitools_exceptions.TransferRetryError, apitools_exceptions.BadStatusCodeError, # TODO: Honor retry-after headers. apitools_exceptions.RetryAfterError, apitools_exceptions.RequestError, http_client.BadStatusLine, http_client.IncompleteRead, http_client.ResponseNotReady, httplib2.ServerNotFoundError, oauth2client.client.HttpAccessTokenRefreshError, socket.error, socket.gaierror, socket.timeout, ssl.SSLError, ValueError, ) # Fields requiring projection=full across all API calls. _ACL_FIELDS_SET = set([ 'acl', 'defaultObjectAcl', 'items/acl', 'items/defaultObjectAcl', 'items/owner', 'owner', ]) _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = { None: None, 'authenticated-read': 'authenticatedRead', 'private': 'private', 'project-private': 'projectPrivate', 'public-read': 'publicRead', 'public-read-write': 'publicReadWrite' } _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = { None: None, 'authenticated-read': 'authenticatedRead', 'bucket-owner-read': 'bucketOwnerRead', 'bucket-owner-full-control': 'bucketOwnerFullControl', 'private': 'private', 'project-private': 'projectPrivate', 'public-read': 'publicRead' } FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION = _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.copy( ) FULL_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION.update( _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION) # Fields that may be encrypted. _ENCRYPTED_HASHES_SET = set(['crc32c', 'md5Hash']) # During listing, if we attempt to decrypt an object but it has been removed, # we skip including the object in the listing. _SKIP_LISTING_OBJECT = 'skip' _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation.') DEFAULT_HOST = 'storage.googleapis.com' MTLS_HOST = 'storage.mtls.googleapis.com' class GcsJsonApi(CloudApi): """Google Cloud Storage JSON implementation of gsutil Cloud API.""" def __init__(self, bucket_storage_uri_class, logger, status_queue, provider=None, credentials=None, debug=0, http_headers=None, trace_token=None, perf_trace_token=None, user_project=None): """Performs necessary setup for interacting with Google Cloud Storage. Args: bucket_storage_uri_class: Unused. logger: logging.logger for outputting log messages. status_queue: Queue for relaying status to UI. provider: Unused. This implementation supports only Google Cloud Storage. credentials: Credentials to be used for interacting with Google Cloud Storage. debug: Debug level for the API implementation (0..3). http_headers (dict\|None): Arbitrary headers to be included in every request. trace_token: Trace token to pass to the API implementation. perf_trace_token: Performance trace token to use when making API calls. user_project: Project to be billed for this request. """ # TODO: Plumb host_header for perfdiag / test_perfdiag. # TODO: Add jitter to apitools' http_wrapper retry mechanism. super(GcsJsonApi, self).__init__(bucket_storage_uri_class, logger, status_queue, provider='gs', debug=debug, http_headers=http_headers, trace_token=trace_token, perf_trace_token=perf_trace_token, user_project=user_project) self.certs_file = GetCertsFile() self.http = GetNewHttp() SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) # Re-use download and upload connections. This class is only called # sequentially, but we can share TCP warmed-up connections across calls. self.download_http = self._GetNewDownloadHttp() self.upload_http = self._GetNewUploadHttp() if self.credentials: self.authorized_download_http = self.credentials.authorize( self.download_http) self.authorized_upload_http = self.credentials.authorize(self.upload_http) else: self.authorized_download_http = self.download_http self.authorized_upload_http = self.upload_http if isP12Credentials(self.credentials): WrapDownloadHttpRequest(self.authorized_download_http.http) WrapUploadHttpRequest(self.authorized_upload_http.http) else: WrapDownloadHttpRequest(self.authorized_download_http) WrapUploadHttpRequest(self.authorized_upload_http) self.http_base = 'https://' gs_json_host = config.get('Credentials', 'gs_json_host', None) if (context_config.get_context_config() and context_config.get_context_config().use_client_certificate): if gs_json_host: raise ArgumentException( '"use_client_certificate" is enabled, which sets gsutil to use the' ' host {}. However, a custom host was set using' ' "gs_json_host": {}. Please set "use_client_certificate" to' ' "False" or comment out the "gs_json_host" line in the Boto' ' config.'.format(MTLS_HOST, gs_json_host)) self.host_base = MTLS_HOST else: self.host_base = gs_json_host or DEFAULT_HOST if not gs_json_host: gs_host = config.get('Credentials', 'gs_host', None) if gs_host: raise ArgumentException( 'JSON API is selected but gs_json_host is not configured, ' 'while gs_host is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_host) gs_json_port = config.get('Credentials', 'gs_json_port', None) if not gs_json_port: gs_port = config.get('Credentials', 'gs_port', None) if gs_port: raise ArgumentException( 'JSON API is selected but gs_json_port is not configured, ' 'while gs_port is configured to %s. Please also configure ' 'gs_json_host and gs_json_port to match your desired endpoint.' % gs_port) self.host_port = '' else: self.host_port = ':' + config.get('Credentials', 'gs_json_port') self.api_version = GetGcsJsonApiVersion() self.url_base = (self.http_base + self.host_base + self.host_port + '/' + 'storage/' + self.api_version + '/') self.global_params = apitools_messages.StandardQueryParameters( trace='token:%s' % trace_token) if trace_token else None additional_http_headers = {} gs_json_host_header = config.get('Credentials', 'gs_json_host_header', None) if gs_json_host_header and gs_json_host: additional_http_headers['Host'] = gs_json_host_header self._UpdateHeaders(additional_http_headers) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.StorageV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials, version=self.api_version, default_global_params=self.global_params, additional_http_headers=additional_http_headers) self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.api_client.retry_func = LogAndHandleRetries( status_queue=self.status_queue) self.api_client.overwrite_transfer_urls_with_client_base = True if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam('key', 'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetServiceAccountId(self): """Returns the service account email id.""" if isinstance(self.credentials, ImpersonationCredentials): return self.credentials.service_account_id elif isinstance(self.credentials, ServiceAccountCredentials): return self.credentials.service_account_email else: raise CommandException( 'Cannot get service account email id for the given ' 'credential type.') def _UpdateHeaders(self, headers): if self.http_headers: headers.update(self.http_headers) # The following functional headers potentially overwrite # arbitrary ones provided by -h. if self.perf_trace_token: headers['cookie'] = self.perf_trace_token request_reason = os.environ.get(REQUEST_REASON_ENV_VAR) if request_reason: headers[REQUEST_REASON_HEADER_KEY] = request_reason def _GetNewDownloadHttp(self): return GetNewHttp(http_class=HttpWithDownloadStream) def _GetNewUploadHttp(self): """Returns an upload-safe Http object (by disabling httplib2 retries).""" return GetNewHttp(http_class=HttpWithNoRetries) def _GetSignedContent(self, string_to_sign): """Returns the Signed Content.""" if isinstance(self.credentials, ImpersonationCredentials): iam_cred_api = self.credentials.api service_account_id = self.credentials.service_account_id response = iam_cred_api.SignBlob(service_account_id, string_to_sign) return response.keyId, response.signedBlob elif isinstance(self.credentials, ServiceAccountCredentials): return self.credentials.sign_blob(string_to_sign) else: raise CommandException( 'Authentication using a service account is required for signing ' 'the content.') def _FieldsContainsAclField(self, fields=None): """Checks Returns true if ACL related values are in fields set. Args: fields: list or set of fields. May be in GET ['acl'] or List ['items/acl'] call format. Returns: True if an ACL value is requested in the input fields, False otherwise. """ return fields is None or _ACL_FIELDS_SET.intersection(set(fields)) def _ValidateEncryptionFieldsForMetadataGetRequest(self, fields=None): """Ensures customerEncryption field is included if hashes are requested.""" # In the context of fetching object metadata, we only need to check for # customerEncryption, the field present if an object is CSEK-encrypted. Note # that if the object is KMS-encrypted, no field checks are necessary; a # metadata get request will be able to access the object's stored kmsKeyName # field to access the key needed to decrypt hash fields). if (fields and _ENCRYPTED_HASHES_SET.intersection(set(fields)) and 'customerEncryption' not in fields): raise ArgumentException( 'gsutil client error: customerEncryption must be included when ' 'requesting potentially encrypted fields %s' % _ENCRYPTED_HASHES_SET) def GetBucketIamPolicy(self, bucket_name, provider=None, fields=None): apitools_request = apitools_messages.StorageBucketsGetIamPolicyRequest( bucket=bucket_name, userProject=self.user_project, optionsRequestedPolicyVersion=IAM_POLICY_VERSION) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.buckets.GetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def GetObjectIamPolicy(self, bucket_name, object_name, generation=None, provider=None, fields=None): if generation is not None: generation = long(generation) apitools_request = apitools_messages.StorageObjectsGetIamPolicyRequest( bucket=bucket_name, object=object_name, generation=generation, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.objects.GetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def SetObjectIamPolicy(self, bucket_name, object_name, policy, generation=None, provider=None): if generation is not None: generation = long(generation) api_request = apitools_messages.StorageObjectsSetIamPolicyRequest( bucket=bucket_name, object=object_name, generation=generation, policy=policy, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.objects.SetIamPolicy(api_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name) def SetBucketIamPolicy(self, bucket_name, policy, provider=None): apitools_request = apitools_messages.StorageBucketsSetIamPolicyRequest( bucket=bucket_name, policy=policy, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.buckets.SetIamPolicy(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def SignUrl(self, method, duration, path, generation, logger, region, signed_headers, string_to_sign_debug): """See CloudApi class for function doc strings.""" service_account_id = self.GetServiceAccountId() string_to_sign, canonical_query_string = CreatePayload( client_id=service_account_id, method=method, duration=duration, path=path, generation=generation, logger=logger, region=region, signed_headers=signed_headers, string_to_sign_debug=string_to_sign_debug) if six.PY3: string_to_sign = string_to_sign.encode(UTF8) key_id, raw_signature = self._GetSignedContent(string_to_sign) logger.debug('Key ID used to sign blob for service account "%s": "%s"' % (service_account_id, key_id)) return GetFinalUrl(raw_signature, signed_headers['host'], path, canonical_query_string) def GetBucket(self, bucket_name, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsGetRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsGetRequest. ProjectionValueValuesEnum.full) apitools_request = apitools_messages.StorageBucketsGetRequest( bucket=bucket_name, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) # Here and in list buckets, we have no way of knowing # whether we requested a field and didn't get it because it didn't exist # or because we didn't have permission to access it. try: return self.api_client.buckets.Get(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListBucketAccessControls(self, bucket_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageBucketAccessControlsListRequest( bucket=bucket_name) return self.api_client.bucketAccessControls.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListObjectAccessControls(self, bucket_name, object_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageObjectAccessControlsListRequest( bucket=bucket_name, object=object_name) return self.api_client.objectAccessControls.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name) def PatchBucket(self, bucket_name, metadata, canned_acl=None, canned_def_acl=None, preconditions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsPatchRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsPatchRequest. ProjectionValueValuesEnum.full) bucket_metadata = metadata if not preconditions: preconditions = Preconditions() # For blank metadata objects, we need to explicitly call # them out to apitools so it will send/erase them. apitools_include_fields = [] for metadata_field in ( 'billing', 'encryption', 'lifecycle', 'logging', 'metadata', 'retentionPolicy', 'versioning', 'website', ): attr = getattr(bucket_metadata, metadata_field, None) if attr and not encoding.MessageToDict(attr): setattr(bucket_metadata, metadata_field, None) apitools_include_fields.append(metadata_field) if bucket_metadata.cors and bucket_metadata.cors == REMOVE_CORS_CONFIG: bucket_metadata.cors = [] apitools_include_fields.append('cors') if (bucket_metadata.defaultObjectAcl and bucket_metadata.defaultObjectAcl[0] == PRIVATE_DEFAULT_OBJ_ACL): bucket_metadata.defaultObjectAcl = [] apitools_include_fields.append('defaultObjectAcl') predefined_acl = None if canned_acl: # Must null out existing ACLs to apply a canned ACL. apitools_include_fields.append('acl') predefined_acl = (apitools_messages.StorageBucketsPatchRequest. PredefinedAclValueValuesEnum( self._BucketCannedAclToPredefinedAcl(canned_acl))) predefined_def_acl = None if canned_def_acl: # Must null out existing default object ACLs to apply a canned ACL. apitools_include_fields.append('defaultObjectAcl') predefined_def_acl = ( apitools_messages.StorageBucketsPatchRequest. PredefinedDefaultObjectAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_def_acl))) apitools_request = apitools_messages.StorageBucketsPatchRequest( bucket=bucket_name, bucketResource=bucket_metadata, projection=projection, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, predefinedDefaultObjectAcl=predefined_def_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) with self.api_client.IncludeFields(apitools_include_fields): try: return self.api_client.buckets.Patch(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def LockRetentionPolicy(self, bucket_name, metageneration, provider=None): try: metageneration = long(metageneration) except ValueError: raise ArgumentException( 'LockRetentionPolicy Metageneration must be an integer.') apitools_request = ( apitools_messages.StorageBucketsLockRetentionPolicyRequest( bucket=bucket_name, ifMetagenerationMatch=metageneration, userProject=self.user_project)) global_params = apitools_messages.StandardQueryParameters() try: return self.api_client.buckets.LockRetentionPolicy( apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def CreateBucket(self, bucket_name, project_id=None, metadata=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsInsertRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsInsertRequest. ProjectionValueValuesEnum.full) if not metadata: metadata = apitools_messages.Bucket() metadata.name = bucket_name if metadata.location: metadata.location = metadata.location.lower() if metadata.storageClass: metadata.storageClass = metadata.storageClass.lower() project_id = PopulateProjectId(project_id) apitools_request = apitools_messages.StorageBucketsInsertRequest( bucket=metadata, project=project_id, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.buckets.Insert(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def DeleteBucket(self, bucket_name, preconditions=None, provider=None): """See CloudApi class for function doc strings.""" if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageBucketsDeleteRequest( bucket=bucket_name, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project) try: self.api_client.buckets.Delete(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if isinstance( self._TranslateApitoolsException(e, bucket_name=bucket_name), NotEmptyException): # If bucket is not empty, check to see if versioning is enabled and # signal that in the exception if it is. bucket_metadata = self.GetBucket(bucket_name, fields=['versioning']) if bucket_metadata.versioning and bucket_metadata.versioning.enabled: raise NotEmptyException('VersionedBucketNotEmpty', status=e.status_code) self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def ListBuckets(self, project_id=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageBucketsListRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageBucketsListRequest. ProjectionValueValuesEnum.full) project_id = PopulateProjectId(project_id) apitools_request = apitools_messages.StorageBucketsListRequest( project=project_id, maxResults=NUM_BUCKETS_PER_LIST_PAGE, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: if 'nextPageToken' not in fields: fields.add('nextPageToken') global_params.fields = ','.join(set(fields)) try: bucket_list = self.api_client.buckets.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for bucket in self._YieldBuckets(bucket_list): yield bucket while bucket_list.nextPageToken: apitools_request = apitools_messages.StorageBucketsListRequest( project=project_id, pageToken=bucket_list.nextPageToken, maxResults=NUM_BUCKETS_PER_LIST_PAGE, projection=projection, userProject=self.user_project) try: bucket_list = self.api_client.buckets.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for bucket in self._YieldBuckets(bucket_list): yield bucket def _YieldBuckets(self, bucket_list): """Yields buckets from a list returned by apitools.""" if bucket_list.items: for bucket in bucket_list.items: yield bucket def ListObjects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsListRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsListRequest. ProjectionValueValuesEnum.full) apitools_request = apitools_messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, maxResults=NUM_OBJECTS_PER_LIST_PAGE, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: fields = set(fields) if 'nextPageToken' not in fields: fields.add('nextPageToken') if ListToGetFields( list_fields=fields).intersection(_ENCRYPTED_HASHES_SET): # We may need to make a follow-up request to decrypt the hashes, # so ensure the required fields are present. fields.add('items/customerEncryption') fields.add('items/kmsKeyName') fields.add('items/generation') fields.add('items/name') global_params.fields = ','.join(fields) list_page = True next_page_token = None while list_page: list_page = False apitools_request = apitools_messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, pageToken=next_page_token, maxResults=NUM_OBJECTS_PER_LIST_PAGE, userProject=self.user_project) try: object_list = self.api_client.objects.List(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) if object_list.nextPageToken: list_page = True next_page_token = object_list.nextPageToken for object_or_prefix in self._YieldObjectsAndPrefixes(object_list): if object_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: decrypted_metadata = self._DecryptHashesIfPossible( bucket_name, object_or_prefix.data, fields=ListToGetFields(fields)) if decrypted_metadata == _SKIP_LISTING_OBJECT: continue elif decrypted_metadata: object_or_prefix.data = decrypted_metadata else: # Object metadata was unencrypted or we did not have a matching # key; yield what we received in the initial listing. pass yield object_or_prefix def _DecryptHashesIfPossible(self, bucket_name, object_metadata, fields=None): """Attempts to decrypt object metadata. Helps fetch encrypted object metadata fields (if requested) for API method calls that requested them but did not receive them; this occurs for ListObjects(), along with GetObjectMetadata() calls for which the object is CSEK-encrypted but no decryption information was supplied. This method issues a StorageObjectsGetRequest (if object hash fields were requested but not retrieved), formatted appropriately depending on the encryption method. Args: bucket_name: String of bucket containing the object. object_metadata: apitools Object describing the object. Must include name, generation, and customerEncryption fields. fields: ListObjects-format fields to return. Returns: - apitools_messages.Object with decrypted hash fields if decryption was performed. - None if the object was not encrypted, or CSEK-encrypted but we could not find a matching decryption key. - gcs_json_api._SKIP_LISTING_OBJECT if the object no longer exists. """ get_fields = ListToGetFields(list_fields=fields) if not self._ObjectIsEncrypted(object_metadata): self._RaiseIfNoDesiredHashesPresentForUnencryptedObject(object_metadata, fields=get_fields) return None # If we reach this point, we know the object is encrypted. get_metadata_func = None if self._ObjectKMSEncryptedAndNeedHashes(object_metadata, fields=get_fields): # For CMK-encrypted objects, we can just call GetObjectMetadata. get_metadata_func = functools.partial( self._GetObjectMetadataHelper, bucket_name, object_metadata.name, generation=object_metadata.generation, fields=get_fields) elif self._ObjectCSEKEncryptedAndNeedHashes(object_metadata, fields=get_fields): # For CSEK-encrypted objects, we can do almost the same thing as above, # but we also need to add some info about the decryption key (if it's # available) into the request. key_sha256 = object_metadata.customerEncryption.keySha256 if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') decryption_key = FindMatchingCSEKInBotoConfig(key_sha256, config) if decryption_key: get_metadata_func = functools.partial( self._GetObjectMetadataHelper, bucket_name, object_metadata.name, generation=object_metadata.generation, fields=get_fields, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) if get_metadata_func is None: return try: return get_metadata_func() except NotFoundException: # If the object was deleted between the listing request and # the get request, do not include it in the list. return _SKIP_LISTING_OBJECT def _YieldObjectsAndPrefixes(self, object_list): # ls depends on iterating objects before prefixes for proper display. if object_list.items: for cloud_obj in object_list.items: yield CloudApi.CsObjectOrPrefix(cloud_obj, CloudApi.CsObjectOrPrefixType.OBJECT) if object_list.prefixes: for prefix in object_list.prefixes: yield CloudApi.CsObjectOrPrefix(prefix, CloudApi.CsObjectOrPrefixType.PREFIX) @contextmanager def _ApitoolsRequestHeaders(self, headers): """Wrapped code makes apitools requests with the specified headers.""" if headers: old_headers = self.api_client.additional_http_headers.copy() self.api_client.additional_http_headers.update(headers) yield if headers: self.api_client.additional_http_headers = old_headers def _EncryptionHeadersFromTuple(self, crypto_tuple=None): """Returns headers dict matching crypto_tuple, or empty dict.""" headers = {} if crypto_tuple and crypto_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-encryption-algorithm'] = crypto_tuple.crypto_alg headers['x-goog-encryption-key'] = crypto_tuple.crypto_key headers['x-goog-encryption-key-sha256'] = ( Base64Sha256FromBase64EncryptionKey(crypto_tuple.crypto_key)) return headers def _RewriteCryptoHeadersFromTuples(self, decryption_tuple=None, encryption_tuple=None): """Returns headers dict matching the provided tuples, or empty dict.""" headers = {} if decryption_tuple and decryption_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-copy-source-encryption-algorithm'] = ( decryption_tuple.crypto_alg) headers['x-goog-copy-source-encryption-key'] = ( decryption_tuple.crypto_key) headers['x-goog-copy-source-encryption-key-sha256'] = ( decryption_tuple.crypto_key_sha256) if encryption_tuple and encryption_tuple.crypto_type == CryptoKeyType.CSEK: headers['x-goog-encryption-algorithm'] = encryption_tuple.crypto_alg headers['x-goog-encryption-key'] = encryption_tuple.crypto_key headers['x-goog-encryption-key-sha256'] = ( encryption_tuple.crypto_key_sha256) return headers def _GetObjectMetadataHelper(self, bucket_name, object_name, generation=None, fields=None, decryption_tuple=None): """Get metadata, adding necessary headers for CSEK decryption if necessary. Args: bucket_name: (str) Bucket containing the object. object_name: (str) Object name. generation: (int) Generation of the object to retrieve. fields: (list<str>) If present, return only these Object metadata fields, for example: ['acl', 'updated']. decryption_tuple: (gslib.utils.encryption_helper.CryptoKeyWrapper) The CryptoKeyWrapper for the desired decryption key. Raises: ArgumentException for errors during input validation. ServiceException for errors interacting with cloud storage providers. Returns: (apitools_message.Object) The retrieved object metadata. """ apitools_request = self._CreateApitoolsObjectMetadataGetRequest( bucket_name, object_name, generation=generation, fields=fields) global_params = self._GetApitoolsObjectMetadataGlobalParams(fields=fields) try: # Add special headers if fetching metadata for a CSEK-encrypted object. if decryption_tuple: with self._ApitoolsRequestHeaders( self._EncryptionHeadersFromTuple(crypto_tuple=decryption_tuple)): return self.api_client.objects.Get(apitools_request, global_params=global_params) else: return self.api_client.objects.Get(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _CreateApitoolsObjectMetadataGetRequest(self, bucket_name, object_name, generation=None, fields=None): self._ValidateEncryptionFieldsForMetadataGetRequest(fields=fields) projection = (apitools_messages.StorageObjectsGetRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsGetRequest. ProjectionValueValuesEnum.full) if generation: generation = long(generation) return apitools_messages.StorageObjectsGetRequest( bucket=bucket_name, object=object_name, projection=projection, generation=generation, userProject=self.user_project) def _GetApitoolsObjectMetadataGlobalParams(self, fields=None): global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) return global_params def GetObjectMetadata(self, bucket_name, object_name, generation=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" try: object_metadata = self._GetObjectMetadataHelper(bucket_name, object_name, generation=generation, fields=fields) if not self._ObjectCSEKEncryptedAndNeedHashes(object_metadata, fields=fields): return object_metadata # CSEK-encrypted objects require corresponding decryption information to # be added to the StorageObjectsGetRequest in order to retrieve hash # fields. key_sha256 = object_metadata.customerEncryption.keySha256 if six.PY3: if not isinstance(key_sha256, bytes): key_sha256 = key_sha256.encode('ascii') decryption_key = FindMatchingCSEKInBotoConfig(key_sha256, config) if not decryption_key: raise EncryptionException( 'Missing decryption key with SHA256 hash %s. No decryption key ' 'matches object %s://%s/%s' % (key_sha256, self.provider, bucket_name, object_name)) if six.PY3: if not isinstance(decryption_key, bytes): decryption_key = decryption_key.encode(UTF8) return self._GetObjectMetadataHelper( bucket_name, object_name, generation=generation, fields=fields, decryption_tuple=CryptoKeyWrapperFromKey(decryption_key)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _ObjectIsEncrypted(self, object_metadata): return bool(object_metadata.customerEncryption or object_metadata.kmsKeyName) def _RaiseIfNoDesiredHashesPresentForUnencryptedObject( self, object_metadata, fields=None): """Raise an exception if none of the desired hash fields are present.""" if fields is None: return md5_requested_but_missing = ('md5Hash' in fields and not object_metadata.md5Hash) crc32c_requested_but_missing = ('crc32c' in fields and not object_metadata.crc32c) # md5 not available for composite objects; in this case, don't raise an # exception if that was the only hash we requested. if (md5_requested_but_missing and not object_metadata.componentCount and crc32c_requested_but_missing): raise ServiceException( 'Service did not provide requested hashes in metadata for ' 'unencrypted object %s' % object_metadata.name) def _ObjectKMSEncryptedAndNeedHashes(self, object_metadata, fields=None): """Checks if an object is KMS-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request should be issued on the KMS-encrypted object in order to retrieve encrypted hash fields, False otherwise. """ if fields is not None and not _ENCRYPTED_HASHES_SET.intersection(fields): # No potentially encrypted metadata fields were requested. return False return (object_metadata.kmsKeyName and not object_metadata.crc32c and not object_metadata.md5Hash) def _ObjectCSEKEncryptedAndNeedHashes(self, object_metadata, fields=None): """Checks if an object is CSEK-encrypted and lacking requested hash fields. Args: object_metadata: (apitools_messages.Object) Object metadata returned from an API call; may or may not contain hash-related metadata fields. fields: (list<str>) Requested object metadata fields. Returns: True if a GetObjectMetadata request including a decryption key should be issued on the CSEK-encrypted object in order to retrieve encrypted hash fields, False otherwise. """ if fields is None or not _ENCRYPTED_HASHES_SET.intersection(fields): # No potentially encrypted metadata fields were requested. return False return (object_metadata.customerEncryption and not object_metadata.crc32c and not object_metadata.md5Hash) def GetObjectMedia(self, bucket_name, object_name, download_stream, provider=None, generation=None, object_size=None, compressed_encoding=False, download_strategy=CloudApi.DownloadStrategy.ONE_SHOT, start_byte=0, end_byte=None, progress_callback=None, serialization_data=None, digesters=None, decryption_tuple=None): """See CloudApi class for function doc strings.""" # This implementation will get the object metadata first if we don't pass it # in via serialization_data. if generation: generation = long(generation) # 'outer_total_size' is only used for formatting user output, and is # expected to be one higher than the last byte that should be downloaded. # TODO: Change DownloadCallbackConnectionClassFactory and progress callbacks # to more elegantly handle total size for components of files. outer_total_size = object_size if end_byte: outer_total_size = end_byte + 1 elif serialization_data: outer_total_size = json.loads( six.ensure_str(serialization_data))['total_size'] if progress_callback: if outer_total_size is None: raise ArgumentException('Download size is required when callbacks are ' 'requested for a download, but no size was ' 'provided.') progress_callback(start_byte, outer_total_size) bytes_downloaded_container = BytesTransferredContainer() bytes_downloaded_container.bytes_transferred = start_byte callback_class_factory = DownloadCallbackConnectionClassFactory( bytes_downloaded_container, total_size=outer_total_size, progress_callback=progress_callback, digesters=digesters) download_http_class = callback_class_factory.GetConnectionClass() # Point our download HTTP at our download stream. self.download_http.stream = download_stream self.download_http.connections = {'https': download_http_class} if serialization_data: # If we have an apiary trace token, add it to the URL. # TODO: Add query parameter support to apitools downloads so there is # a well-defined way to express query parameters. Currently, we assume # the URL ends in ?alt=media, and this will break if that changes. if self.trace_token: serialization_dict = json.loads(six.ensure_str(serialization_data)) serialization_dict['url'] += '&trace=token%%3A%s' % self.trace_token serialization_data = json.dumps(serialization_dict) apitools_download = apitools_transfer.Download.FromData( download_stream, serialization_data, self.api_client.http, num_retries=self.num_retries, client=self.api_client) else: apitools_download = apitools_transfer.Download.FromStream( download_stream, auto_transfer=False, total_size=object_size, num_retries=self.num_retries) apitools_download.bytes_http = self.authorized_download_http apitools_request = apitools_messages.StorageObjectsGetRequest( bucket=bucket_name, object=object_name, generation=generation, userProject=self.user_project) # Disable retries in apitools. We will handle them explicitly for # resumable downloads; one-shot downloads are not retryable as we do # not track how many bytes were written to the stream. apitools_download.retry_func = LogAndHandleRetries( is_data_transfer=True, status_queue=self.status_queue) try: if download_strategy == CloudApi.DownloadStrategy.RESUMABLE: return self._PerformResumableDownload( bucket_name, object_name, download_stream, apitools_request, apitools_download, bytes_downloaded_container, compressed_encoding=compressed_encoding, generation=generation, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) else: return self._PerformDownload(bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=generation, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) # If you are fighting a redacted exception spew in multiprocess/multithread # calls, add your exception to TRANSLATABLE_APITOOLS_EXCEPTIONS and put # something like this immediately after the following except statement: # import sys, traceback; sys.stderr.write('\n{}\n'.format( # traceback.format_exc())); sys.stderr.flush() # This may hang, but you should get a stack trace spew after Ctrl-C. except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _PerformResumableDownload(self, bucket_name, object_name, download_stream, apitools_request, apitools_download, bytes_downloaded_container, generation=None, compressed_encoding=False, start_byte=0, end_byte=None, serialization_data=None, decryption_tuple=None): retries = 0 last_progress_byte = start_byte while retries <= self.num_retries: try: return self._PerformDownload(bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=generation, compressed_encoding=compressed_encoding, start_byte=start_byte, end_byte=end_byte, serialization_data=serialization_data, decryption_tuple=decryption_tuple) except HTTP_TRANSFER_EXCEPTIONS as e: self._ValidateHttpAccessTokenRefreshError(e) start_byte = download_stream.tell() bytes_downloaded_container.bytes_transferred = start_byte if start_byte > last_progress_byte: # We've made progress, so allow a fresh set of retries. last_progress_byte = start_byte retries = 0 retries += 1 if retries > self.num_retries: ##### DEBUG # If you are fighting a redacted exception spew in # multiprocess/multithread calls, add your exception to # HTTP_TRANSFER_EXCEPTIONS and uncomment the following block. This # may hang, but you should get a stack trace spew after Ctrl-C. ##### # import re, sys, traceback # from gslib.utils import text_util # message = 'some exception happened' # stack_trace = traceback.format_exc() # err = ('DEBUG: Exception stack trace:\n %s\n%s\n' % ( # re.sub('\\n', '\n ', stack_trace), # message, # )) # dest_fd = sys.stderr # text_util.print_to_fd(err, end='', file=dest_fd) # dest_fd.flush() ##### END DEBUG raise ResumableDownloadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, GetPrintableExceptionString(e))) time.sleep(CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug( 'Retrying download from byte %s after exception: %s. Trace: %s', start_byte, GetPrintableExceptionString(e), traceback.format_exc()) apitools_http_wrapper.RebuildHttpConnections( apitools_download.bytes_http) def _PerformDownload(self, bucket_name, object_name, download_stream, apitools_request, apitools_download, generation=None, compressed_encoding=False, start_byte=0, end_byte=None, serialization_data=None, decryption_tuple=None): if not serialization_data: try: self.api_client.objects.Get(apitools_request, download=apitools_download) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) # Disable apitools' default print callbacks. def _NoOpCallback(unused_response, unused_download_object): pass # TODO: If we have a resumable download with accept-encoding:gzip # on a object that is compressible but not in gzip form in the cloud, # on-the-fly compression may gzip the object. In this case if our # download breaks, future requests will ignore the range header and just # return the object (gzipped) in its entirety. Ideally, we would unzip # the bytes that we have locally and send a range request without # accept-encoding:gzip so that we can download only the (uncompressed) bytes # that we don't yet have. # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } AddAcceptEncodingGzipIfNeeded(additional_headers, compressed_encoding=compressed_encoding) self._UpdateHeaders(additional_headers) additional_headers.update( self._EncryptionHeadersFromTuple(decryption_tuple)) if start_byte or end_byte is not None: apitools_download.GetRange(additional_headers=additional_headers, start=start_byte, end=end_byte, use_chunks=False) else: apitools_download.StreamMedia(callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=additional_headers, use_chunks=False) return apitools_download.encoding def PatchObjectMetadata(self, bucket_name, object_name, metadata, canned_acl=None, generation=None, preconditions=None, provider=None, fields=None, user_project=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsPatchRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsPatchRequest. ProjectionValueValuesEnum.full) if not preconditions: preconditions = Preconditions() if generation: generation = long(generation) predefined_acl = None apitools_include_fields = [] if canned_acl: # Must null out existing ACLs to apply a canned ACL. apitools_include_fields.append('acl') predefined_acl = (apitools_messages.StorageObjectsPatchRequest. PredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) # Provide the ability to delete response headers from metadata. if metadata.cacheControl == '': apitools_include_fields.append('cacheControl') if metadata.contentDisposition == '': apitools_include_fields.append('contentDisposition') if metadata.contentEncoding == '': apitools_include_fields.append('contentEncoding') if metadata.contentLanguage == '': apitools_include_fields.append('contentLanguage') apitools_request = apitools_messages.StorageObjectsPatchRequest( bucket=bucket_name, object=object_name, objectResource=metadata, generation=generation, projection=projection, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: with self.api_client.IncludeFields(apitools_include_fields): return self.api_client.objects.Patch(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def _UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, encryption_tuple=None, provider=None, fields=None, serialization_data=None, tracker_callback=None, progress_callback=None, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, total_size=0, gzip_encoded=False): # pylint: disable=g-doc-args """Upload implementation. Cloud API arguments, plus two more. Additional args: apitools_strategy: SIMPLE_UPLOAD or RESUMABLE_UPLOAD. total_size: Total size of the upload; None if it is unknown (streaming). Returns: Uploaded object metadata. """ # pylint: enable=g-doc-args ValidateDstObjectMetadata(object_metadata) predefined_acl = None if canned_acl: predefined_acl = (apitools_messages.StorageObjectsInsertRequest. PredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) bytes_uploaded_container = BytesTransferredContainer() if progress_callback and size: total_size = size progress_callback(0, size) callback_class_factory = UploadCallbackConnectionClassFactory( bytes_uploaded_container, total_size=total_size, progress_callback=progress_callback, logger=self.logger, debug=self.debug) upload_http_class = callback_class_factory.GetConnectionClass() self.upload_http.connections = { 'http': upload_http_class, 'https': upload_http_class, } # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } self._UpdateHeaders(additional_headers) try: content_type = None apitools_request = None global_params = None if not serialization_data: # This is a new upload, set up initial upload state. content_type = object_metadata.contentType if not content_type: content_type = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() apitools_request = apitools_messages.StorageObjectsInsertRequest( bucket=object_metadata.bucket, object=object_metadata, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, predefinedAcl=predefined_acl, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) encryption_headers = self._EncryptionHeadersFromTuple( crypto_tuple=encryption_tuple) if apitools_strategy == apitools_transfer.SIMPLE_UPLOAD: # One-shot upload. apitools_upload = apitools_transfer.Upload(upload_stream, content_type, total_size=size, auto_transfer=True, num_retries=self.num_retries, gzip_encoded=gzip_encoded) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = self.authorized_upload_http with self._ApitoolsRequestHeaders(encryption_headers): return self.api_client.objects.Insert(apitools_request, upload=apitools_upload, global_params=global_params) else: # Resumable upload. # Since bytes_http is created in this function, we don't get the # user-agent header from api_client's http automatically. additional_headers = { 'user-agent': self.api_client.user_agent, } additional_headers.update(encryption_headers) self._UpdateHeaders(additional_headers) return self._PerformResumableUpload( upload_stream, self.authorized_upload_http, content_type, size, serialization_data, apitools_strategy, apitools_request, global_params, bytes_uploaded_container, tracker_callback, additional_headers, progress_callback, gzip_encoded) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, object_metadata.bucket) self._TranslateExceptionAndRaise(e, bucket_name=object_metadata.bucket, object_name=object_metadata.name, not_found_exception=not_found_exception) def _PerformResumableUpload(self, upload_stream, authorized_upload_http, content_type, size, serialization_data, apitools_strategy, apitools_request, global_params, bytes_uploaded_container, tracker_callback, addl_headers, progress_callback, gzip_encoded): try: if serialization_data: # Resuming an existing upload. apitools_upload = apitools_transfer.Upload.FromData( upload_stream, serialization_data, self.api_client.http, num_retries=self.num_retries, gzip_encoded=gzip_encoded, client=self.api_client) apitools_upload.chunksize = GetJsonResumableChunkSize() apitools_upload.bytes_http = authorized_upload_http else: # New resumable upload. apitools_upload = apitools_transfer.Upload( upload_stream, content_type, total_size=size, chunksize=GetJsonResumableChunkSize(), auto_transfer=False, num_retries=self.num_retries, gzip_encoded=gzip_encoded) apitools_upload.strategy = apitools_strategy apitools_upload.bytes_http = authorized_upload_http with self._ApitoolsRequestHeaders(addl_headers): self.api_client.objects.Insert(apitools_request, upload=apitools_upload, global_params=global_params) # Disable retries in apitools. We will handle them explicitly here. apitools_upload.retry_func = LogAndHandleRetries( is_data_transfer=True, status_queue=self.status_queue) # Disable apitools' default print callbacks. def _NoOpCallback(unused_response, unused_upload_object): pass # If we're resuming an upload, apitools has at this point received # from the server how many bytes it already has. Update our # callback class with this information. bytes_uploaded_container.bytes_transferred = apitools_upload.progress if tracker_callback: tracker_callback(json.dumps(apitools_upload.serialization_data)) retries = 0 last_progress_byte = apitools_upload.progress while retries <= self.num_retries: try: # TODO: On retry, this will seek to the bytes that the server has, # causing the hash to be recalculated. Make HashingFileUploadWrapper # save a digest according to json_resumable_chunk_size. if not gzip_encoded and size and not JsonResumableChunkSizeDefined(): # If size is known and the request doesn't need to be compressed, # we can send it all in one request and avoid making a # round-trip per chunk. Compression is not supported for # non-chunked streaming uploads because supporting resumability # for that feature results in degraded upload performance and # adds significant complexity to the implementation. http_response = apitools_upload.StreamMedia( callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=addl_headers) else: # Otherwise it's a streaming request and we need to ensure that we # send the bytes in chunks so that we can guarantee that we never # need to seek backwards more than our buffer (and also that the # chunks are aligned to 256KB). http_response = apitools_upload.StreamInChunks( callback=_NoOpCallback, finish_callback=_NoOpCallback, additional_headers=addl_headers) processed_response = self.api_client.objects.ProcessHttpResponse( self.api_client.objects.GetMethodConfig('Insert'), http_response) if size is None and progress_callback: # Make final progress callback; total size should now be known. # This works around the fact the send function counts header bytes. # However, this will make the progress appear to go slightly # backwards at the end. progress_callback(apitools_upload.total_size, apitools_upload.total_size) return processed_response except HTTP_TRANSFER_EXCEPTIONS as e: self._ValidateHttpAccessTokenRefreshError(e) apitools_http_wrapper.RebuildHttpConnections( apitools_upload.bytes_http) while retries <= self.num_retries: try: # TODO: Simulate the refresh case in tests. Right now, our # mocks are not complex enough to simulate a failure. apitools_upload.RefreshResumableUploadState() start_byte = apitools_upload.progress bytes_uploaded_container.bytes_transferred = start_byte break except HTTP_TRANSFER_EXCEPTIONS as e2: self._ValidateHttpAccessTokenRefreshError(e2) apitools_http_wrapper.RebuildHttpConnections( apitools_upload.bytes_http) retries += 1 if retries > self.num_retries: raise ResumableUploadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, e2)) time.sleep( CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if start_byte > last_progress_byte: # We've made progress, so allow a fresh set of retries. last_progress_byte = start_byte retries = 0 else: retries += 1 if retries > self.num_retries: raise ResumableUploadException( 'Transfer failed after %d retries. Final exception: %s' % (self.num_retries, GetPrintableExceptionString(e))) time.sleep( CalculateWaitForRetry(retries, max_wait=self.max_retry_wait)) if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug( 'Retrying upload from byte %s after exception: %s. Trace: %s', start_byte, GetPrintableExceptionString(e), traceback.format_exc()) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: resumable_ex = self._TranslateApitoolsResumableUploadException(e) if resumable_ex: raise resumable_ex else: raise def UploadObject(self, upload_stream, object_metadata, canned_acl=None, size=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" return self._UploadObject(upload_stream, object_metadata, canned_acl=canned_acl, size=size, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, gzip_encoded=gzip_encoded) def UploadObjectStreaming(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" # Streaming indicated by not passing a size. # Resumable capabilities are present up to the resumable chunk size using # a buffered stream. return self._UploadObject( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, progress_callback=progress_callback, encryption_tuple=encryption_tuple, fields=fields, apitools_strategy=apitools_transfer.RESUMABLE_UPLOAD, total_size=None, gzip_encoded=gzip_encoded) def UploadObjectResumable(self, upload_stream, object_metadata, canned_acl=None, preconditions=None, size=None, serialization_data=None, tracker_callback=None, progress_callback=None, encryption_tuple=None, provider=None, fields=None, gzip_encoded=False): """See CloudApi class for function doc strings.""" return self._UploadObject( upload_stream, object_metadata, canned_acl=canned_acl, preconditions=preconditions, fields=fields, size=size, serialization_data=serialization_data, tracker_callback=tracker_callback, progress_callback=progress_callback, encryption_tuple=encryption_tuple, apitools_strategy=apitools_transfer.RESUMABLE_UPLOAD, gzip_encoded=gzip_encoded) def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None, canned_acl=None, preconditions=None, progress_callback=None, max_bytes_per_call=None, encryption_tuple=None, decryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" ValidateDstObjectMetadata(dst_obj_metadata) predefined_acl = None if canned_acl: predefined_acl = (apitools_messages.StorageObjectsRewriteRequest. DestinationPredefinedAclValueValuesEnum( self._ObjectCannedAclToPredefinedAcl(canned_acl))) if src_generation: src_generation = long(src_generation) if not preconditions: preconditions = Preconditions() projection = (apitools_messages.StorageObjectsRewriteRequest. ProjectionValueValuesEnum.noAcl) if self._FieldsContainsAclField(fields): projection = (apitools_messages.StorageObjectsRewriteRequest. ProjectionValueValuesEnum.full) global_params = apitools_messages.StandardQueryParameters() if fields: # Rewrite returns the resultant object under the 'resource' field. new_fields = set([ 'done', 'objectSize', 'rewriteToken', 'totalBytesRewritten', ]) for field in fields: new_fields.add('resource/' + field) global_params.fields = ','.join(set(new_fields)) dec_key_sha256 = None if decryption_tuple and decryption_tuple.crypto_type == CryptoKeyType.CSEK: dec_key_sha256 = decryption_tuple.crypto_key_sha256 enc_key_sha256 = None if encryption_tuple: if encryption_tuple.crypto_type == CryptoKeyType.CSEK: enc_key_sha256 = encryption_tuple.crypto_key_sha256 elif encryption_tuple.crypto_type == CryptoKeyType.CMEK: dst_obj_metadata.kmsKeyName = encryption_tuple.crypto_key # Check to see if we are resuming a rewrite. tracker_file_name = GetRewriteTrackerFilePath(src_obj_metadata.bucket, src_obj_metadata.name, dst_obj_metadata.bucket, dst_obj_metadata.name, 'JSON') rewrite_params_hash = HashRewriteParameters( src_obj_metadata, dst_obj_metadata, projection, src_generation=src_generation, gen_match=preconditions.gen_match, meta_gen_match=preconditions.meta_gen_match, canned_acl=predefined_acl, max_bytes_per_call=max_bytes_per_call, src_dec_key_sha256=dec_key_sha256, dst_enc_key_sha256=enc_key_sha256, fields=global_params.fields) resume_rewrite_token = ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash) crypto_headers = self._RewriteCryptoHeadersFromTuples( decryption_tuple=decryption_tuple, encryption_tuple=encryption_tuple) progress_cb_with_timeout = None try: last_bytes_written = long(0) while True: with self._ApitoolsRequestHeaders(crypto_headers): apitools_request = apitools_messages.StorageObjectsRewriteRequest( sourceBucket=src_obj_metadata.bucket, sourceObject=src_obj_metadata.name, destinationBucket=dst_obj_metadata.bucket, # TODO(KMS): Remove the destinationKmsKeyName parameter once the # API begins pulling key name from the dest obj metadata. destinationKmsKeyName=dst_obj_metadata.kmsKeyName, destinationObject=dst_obj_metadata.name, projection=projection, object=dst_obj_metadata, sourceGeneration=src_generation, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, destinationPredefinedAcl=predefined_acl, rewriteToken=resume_rewrite_token, maxBytesRewrittenPerCall=max_bytes_per_call, userProject=self.user_project) rewrite_response = self.api_client.objects.Rewrite( apitools_request, global_params=global_params) bytes_written = long(rewrite_response.totalBytesRewritten) if progress_callback and not progress_cb_with_timeout: progress_cb_with_timeout = ProgressCallbackWithTimeout( long(rewrite_response.objectSize), progress_callback) if progress_cb_with_timeout: progress_cb_with_timeout.Progress(bytes_written - last_bytes_written) if rewrite_response.done: break elif not resume_rewrite_token: # Save the token and make a tracker file if they don't already exist. resume_rewrite_token = rewrite_response.rewriteToken WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash, rewrite_response.rewriteToken) last_bytes_written = bytes_written DeleteTrackerFile(tracker_file_name) return rewrite_response.resource except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: not_found_exception = CreateNotFoundExceptionForObjectWrite( self.provider, dst_obj_metadata.bucket, src_provider=self.provider, src_bucket_name=src_obj_metadata.bucket, src_object_name=src_obj_metadata.name, src_generation=src_generation) self._TranslateExceptionAndRaise(e, bucket_name=dst_obj_metadata.bucket, object_name=dst_obj_metadata.name, not_found_exception=not_found_exception) def DeleteObject(self, bucket_name, object_name, preconditions=None, generation=None, provider=None): """See CloudApi class for function doc strings.""" if not preconditions: preconditions = Preconditions() if generation: generation = long(generation) apitools_request = apitools_messages.StorageObjectsDeleteRequest( bucket=bucket_name, object=object_name, generation=generation, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project) try: return self.api_client.objects.Delete(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name, object_name=object_name, generation=generation) def ComposeObject(self, src_objs_metadata, dst_obj_metadata, preconditions=None, encryption_tuple=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" ValidateDstObjectMetadata(dst_obj_metadata) dst_obj_name = dst_obj_metadata.name dst_obj_metadata.name = None dst_bucket_name = dst_obj_metadata.bucket dst_obj_metadata.bucket = None if not dst_obj_metadata.contentType: dst_obj_metadata.contentType = DEFAULT_CONTENT_TYPE if not preconditions: preconditions = Preconditions() global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) src_objs_compose_request = apitools_messages.ComposeRequest( sourceObjects=src_objs_metadata, destination=dst_obj_metadata) encryption_headers = self._EncryptionHeadersFromTuple( crypto_tuple=encryption_tuple) kmsKeyName = None if encryption_tuple: if encryption_tuple.crypto_type == CryptoKeyType.CMEK: kmsKeyName = encryption_tuple.crypto_key with self._ApitoolsRequestHeaders(encryption_headers): apitools_request = apitools_messages.StorageObjectsComposeRequest( composeRequest=src_objs_compose_request, destinationBucket=dst_bucket_name, destinationObject=dst_obj_name, ifGenerationMatch=preconditions.gen_match, ifMetagenerationMatch=preconditions.meta_gen_match, userProject=self.user_project, kmsKeyName=kmsKeyName) try: return self.api_client.objects.Compose(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: # We can't be sure which object was missing in the 404 case. if (isinstance(e, apitools_exceptions.HttpError) and e.status_code == 404): raise NotFoundException('One of the source objects does not exist.') else: self._TranslateExceptionAndRaise(e) def WatchBucket(self, bucket_name, address, channel_id, token=None, provider=None, fields=None): """See CloudApi class for function doc strings.""" projection = (apitools_messages.StorageObjectsWatchAllRequest. ProjectionValueValuesEnum.full) channel = apitools_messages.Channel(address=address, id=channel_id, token=token, type='WEB_HOOK') apitools_request = apitools_messages.StorageObjectsWatchAllRequest( bucket=bucket_name, channel=channel, projection=projection, userProject=self.user_project) global_params = apitools_messages.StandardQueryParameters() if fields: global_params.fields = ','.join(set(fields)) try: return self.api_client.objects.WatchAll(apitools_request, global_params=global_params) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def StopChannel(self, channel_id, resource_id, provider=None): """See CloudApi class for function doc strings.""" channel = apitools_messages.Channel(id=channel_id, resourceId=resource_id) try: self.api_client.channels.Stop(channel) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListChannels(self, bucket_name, provider=None, fields=None): """See CloudApi class for function doc strings.""" apitools_request = apitools_messages.StorageBucketsListChannelsRequest( bucket=bucket_name, userProject=self.user_project) try: return self.api_client.buckets.ListChannels(apitools_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, bucket_name=bucket_name) def GetProjectServiceAccount(self, project_number): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsServiceAccountGetRequest( projectId=six.text_type(project_number)) return self.api_client.projects_serviceAccount.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def CreateNotificationConfig(self, bucket_name, pubsub_topic, payload_format, event_types=None, custom_attributes=None, object_name_prefix=None): """See CloudApi class for function doc strings.""" try: notification = apitools_messages.Notification( topic=pubsub_topic, payload_format=payload_format) if event_types: notification.event_types = event_types if custom_attributes: additional_properties = [ NotificationAdditionalProperty(key=key, value=value) for key, value in custom_attributes.items() ] notification.custom_attributes = NotificationCustomAttributesValue( additionalProperties=additional_properties) if object_name_prefix: notification.object_name_prefix = object_name_prefix request = apitools_messages.StorageNotificationsInsertRequest( bucket=bucket_name, notification=notification, userProject=self.user_project) return self.api_client.notifications.Insert(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def DeleteNotificationConfig(self, bucket_name, notification): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageNotificationsDeleteRequest( bucket=bucket_name, notification=notification, userProject=self.user_project) return self.api_client.notifications.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListNotificationConfigs(self, bucket_name): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageNotificationsListRequest( bucket=bucket_name, userProject=self.user_project) response = self.api_client.notifications.List(request) for notification in response.items: yield notification except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def CreateHmacKey(self, project_id, service_account_email): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysCreateRequest( projectId=project_id, serviceAccountEmail=service_account_email) return self.api_client.hmacKeys.Create(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def DeleteHmacKey(self, project_id, access_id): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysDeleteRequest( projectId=project_id, accessId=access_id) return self.api_client.hmacKeys.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def GetHmacKey(self, project_id, access_id): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysGetRequest( projectId=project_id, accessId=access_id) return self.api_client.hmacKeys.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def ListHmacKeys(self, project_id, service_account_email, show_deleted_keys=True): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysListRequest( projectId=project_id, serviceAccountEmail=service_account_email, showDeletedKeys=show_deleted_keys, maxResults=NUM_BUCKETS_PER_LIST_PAGE) response = self.api_client.hmacKeys.List(request) for key in response.items: yield key except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) while response.nextPageToken: request = apitools_messages.StorageProjectsHmacKeysListRequest( projectId=project_id, serviceAccountEmail=service_account_email, showDeletedKeys=show_deleted_keys, pageToken=response.nextPageToken, maxResults=NUM_BUCKETS_PER_LIST_PAGE) try: response = self.api_client.hmacKeys.List(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) for key in response.items: yield key def UpdateHmacKey(self, project_id, access_id, state, etag): """See CloudApi class for function doc strings.""" try: request = apitools_messages.StorageProjectsHmacKeysUpdateRequest( projectId=project_id, accessId=access_id) metadata = apitools_messages.HmacKeyMetadata() metadata.state = state if etag: metadata.etag = etag request.resource = metadata return self.api_client.hmacKeys.Update(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e) def _BucketCannedAclToPredefinedAcl(self, canned_acl_string): """Translates the input string to a bucket PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect bucket access controls. """ if canned_acl_string in _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION: return _BUCKET_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[canned_acl_string] raise ArgumentException('Invalid canned ACL %s' % canned_acl_string) def _ObjectCannedAclToPredefinedAcl(self, canned_acl_string): """Translates the input string to an object PredefinedAcl string. Args: canned_acl_string: Canned ACL string. Returns: String that can be used as a query parameter with the JSON API. This corresponds to a flavor of PredefinedAclValueValuesEnum and can be used as input to apitools requests that affect object access controls. """ if canned_acl_string in _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION: return _OBJECT_PREDEFINED_ACL_XML_TO_JSON_TRANSLATION[canned_acl_string] raise ArgumentException('Invalid canned ACL %s' % canned_acl_string) def _ValidateHttpAccessTokenRefreshError(self, e): if (isinstance(e, oauth2client.client.HttpAccessTokenRefreshError) and not (e.status == 429 or e.status >= 500)): raise def _TranslateExceptionAndRaise(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, bucket_name=bucket_name, object_name=object_name, generation=generation, not_found_exception=not_found_exception) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(six.ensure_str(http_error.content)) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, scope="${space separated list of acceptable # scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsResumableUploadException(self, e): if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code >= 500: return ResumableUploadException(message or 'Server Error', status=e.status_code) elif e.status_code == 429: return ResumableUploadException(message or 'Too Many Requests', status=e.status_code) elif e.status_code == 410: return ResumableUploadStartOverException(message or 'Bad Request', status=e.status_code) elif e.status_code == 404: return ResumableUploadStartOverException(message or 'Bad Request', status=e.status_code) elif e.status_code >= 400: return ResumableUploadAbortException(message or 'Bad Request', status=e.status_code) if isinstance(e, apitools_exceptions.StreamExhausted): return ResumableUploadAbortException( '%s; if this issue persists, try deleting the tracker files present' ' under ~/.gsutil/tracker-files/' % str(e)) if isinstance(e, apitools_exceptions.TransferError): if ('Aborting transfer' in str(e) or 'Not enough bytes in stream' in str(e)): return ResumableUploadAbortException(str(e)) elif 'additional bytes left in stream' in str(e): return ResumableUploadAbortException( '%s; this can happen if a file changes size while being uploaded' % str(e)) def _TranslateApitoolsException(self, e, bucket_name=None, object_name=None, generation=None, not_found_exception=None): """Translates apitools exceptions into their gsutil Cloud Api equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. bucket_name: Optional bucket name in request that caused the exception. object_name: Optional object name in request that caused the exception. generation: Optional generation in request that caused the exception. not_found_exception: Optional exception to raise in the not-found case. Returns: ServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'The bucket you tried to delete was not empty.' in str(e): return NotEmptyException('BucketNotEmpty (%s)' % bucket_name, status=e.status_code) elif ('The bucket you tried to create requires domain ownership ' 'verification.' in str(e)): return AccessDeniedException( 'The bucket you tried to create requires domain ownership ' 'verification. Please see ' 'https://cloud.google.com/storage/docs/naming' '?hl=en#verification for more details.', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs and Auth and enable the ' 'Google Cloud Storage JSON API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif 'does not have permission to publish messages' in str(e): return PublishPermissionDeniedException(message, status=e.status_code) else: return AccessDeniedException(message or str(e), status=e.status_code) elif e.status_code == 404: if not_found_exception: # The exception is pre-constructed prior to translation; the HTTP # status code isn't available at that time. setattr(not_found_exception, 'status', e.status_code) return not_found_exception elif bucket_name: if object_name: return CreateObjectNotFoundException(e.status_code, self.provider, bucket_name, object_name, generation=generation) return CreateBucketNotFoundException(e.status_code, self.provider, bucket_name) return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and bucket_name: if 'The bucket you tried to delete is not empty.' in str(e): return NotEmptyException('BucketNotEmpty (%s)' % bucket_name, status=e.status_code) return ServiceException( 'A Cloud Storage bucket named \'%s\' already exists. Try another ' 'name. Bucket names must be globally unique across all Google Cloud ' 'projects, including those outside of your ' 'organization.' % bucket_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) elif isinstance(e, apitools_exceptions.TransferInvalidError): return ServiceException('Transfer invalid (possible encoding error: %s)' % str(e))	104.02 KB	2025-06-25 08:19
📄 gcs_json_credentials.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Credentials logic for JSON CloudApi implementation.""" # This module duplicates some logic in third_party gcs_oauth2_boto_plugin # because apitools credentials lib has its own mechanisms for file-locking # and credential storage. As such, it doesn't require most of the # gcs_oauth2_boto_plugin logic. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import base64 import json import logging import os import io import six import traceback # pylint: disable=g-bad-import-order from apitools.base.py import credentials_lib from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cred_types import CredTypes from gslib.exception import CommandException from gslib.impersonation_credentials import ImpersonationCredentials from gslib.no_op_credentials import NoOpCredentials from gslib.utils import constants from gslib.utils import system_util from gslib.utils.boto_util import GetFriendlyConfigFilePaths from gslib.utils.boto_util import GetCredentialStoreFilename from gslib.utils.boto_util import GetGceCredentialCacheFilename from gslib.utils.boto_util import GetGcsJsonApiVersion from gslib.utils.constants import UTF8 from gslib.utils.wrapped_credentials import WrappedCredentials from google.auth import _helpers from google.auth.crypt import base as crypt_base from google_auth_httplib2 import AuthorizedHttp from google.oauth2 import service_account import oauth2client from oauth2client.client import HAS_CRYPTO from oauth2client.contrib import devshell from oauth2client.service_account import ServiceAccountCredentials from google_reauth import reauth_creds from oauth2client.contrib import multiprocess_file_storage from six import BytesIO DEFAULT_GOOGLE_OAUTH2_PROVIDER_AUTHORIZATION_URI = ( 'https://accounts.google.com/o/oauth2/auth') DEFAULT_GOOGLE_OAUTH2_PROVIDER_TOKEN_URI = ( 'https://oauth2.googleapis.com/token') DEFAULT_SCOPES = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.CLOUD_PLATFORM_READ_ONLY, constants.Scopes.FULL_CONTROL, constants.Scopes.READ_ONLY, constants.Scopes.READ_WRITE, ] GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD = 'notasecret' def isP12Credentials(credentials): return isinstance(credentials, P12Credentials) class PKCS12Signer(crypt_base.Signer, crypt_base.FromServiceAccountMixin): """Signer for a p12 service account key.""" def __init__(self, key): self._key = key # Defined in crypt_base.Signer interface. # It is not used by p12 service account keys. @property def key_id(self): return None def sign(self, message): message = _helpers.to_bytes(message) from google.auth.crypt import _cryptography_rsa # pylint: disable=g-import-not-at-top return self._key.sign( message, _cryptography_rsa._PADDING, # pylint: disable=protected-access _cryptography_rsa._SHA256) # pylint: disable=protected-access @classmethod def from_string(cls, key_strings, key_id=None): del key_id #Unused key_string, password = (_helpers.to_bytes(k) for k in key_strings) # Cryptography package is not bundled with gsutil, Keeping it at top would throw error. from cryptography.hazmat.primitives.serialization import pkcs12 # pylint: disable=g-import-not-at-top try: key, _, _ = pkcs12.load_key_and_certificates(key_string, password) return cls(key) except: raise CommandException('Unable to load the keyfile, Invalid password or PKCS12 data.') class P12Credentials(service_account.Credentials): """google-auth service account credentials for p12 keys. p12 keys are not supported by the google-auth service account credentials. gsutil uses oauth2client to support p12 key users. Since oauth2client was deprecated and bundling it is security concern, we decided to support p12 in gsutil codebase. We prefer not adding it to the google-auth library because p12 is not supported from the beginning by google-auth. GCP strongly suggests users to use the JSON format. gsutil has to support it to not break users. """ _REQUIRED_FIELDS = ('service_account_email', 'token_uri', 'scopes') def authorize(self, http): return AuthorizedHttp(self, http=http) @classmethod def from_service_account_pkcs12_keystring(cls, key_string, password=None, kwargs): password = password or GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD signer = PKCS12Signer.from_string((key_string, password)) missing_fields = [f for f in cls._REQUIRED_FIELDS if f not in kwargs] if missing_fields: raise CommandException('Missing fields: {}.'.format( ', '.join(missing_fields))) creds = cls(signer, kwargs) return creds def CreateP12ServiceAccount(key_string, password=None, kwargs): """Creates a service account from a p12 key and handles import errors.""" try: return P12Credentials.from_service_account_pkcs12_keystring( key_string, password, kwargs) except ImportError: raise CommandException( ('pyca/cryptography is not available. Either install it, or please consider using the .json keyfile')) def GetCredentialStoreKey(credentials, api_version): """Disambiguates a credential for caching in a credential store. Different credential types have different fields that identify them. This function assembles relevant information in a string to be used as the key for accessing a credential. Note that in addition to uniquely identifying the entity to which a credential corresponds, we must differentiate between two or more of that entity's credentials that have different attributes such that the credentials should not be treated as interchangeable, e.g. if they target different API versions (happens for developers targeting different test environments), have different private key IDs (for service account JSON keyfiles), or target different provider token (refresh) URIs. Args: credentials: An OAuth2Credentials object. api_version: JSON API version being used. Returns: A string that can be used as the key to identify a credential, e.g. "v1-909320924072.apps.googleusercontent.com-1/rEfrEshtOkEn-https://..." """ # Note: We don't include the scopes as part of the key. For a user credential # object, we always construct it with manually added scopes that are necessary # to negotiate reauth challenges - those scopes don't necessarily correspond # to the scopes the refresh token was created with. We avoid key name # mismatches for the same refresh token by not including scopes in the key # string. key_parts = [api_version] if isinstance(credentials, devshell.DevshellCredentials): key_parts.append(credentials.user_email) elif isinstance(credentials, ServiceAccountCredentials): # pylint: disable=protected-access key_parts.append(credentials._service_account_email) if getattr(credentials, '_private_key_id', None): # JSON keyfile. # Differentiate between two different JSON keyfiles for the same service # account. key_parts.append(credentials._private_key_id) elif getattr(credentials, '_private_key_pkcs12', None): # P12 keyfile # Use a prefix of the Base64-encoded PEM string to differentiate it from # others. Using a prefix of reasonable length prevents the key from being # unnecessarily large, and the likelihood of having two PEM strings with # the same prefixes is sufficiently low. key_parts.append(base64.b64encode(credentials._private_key_pkcs12)[:20]) # pylint: enable=protected-access elif isinstance(credentials, oauth2client.client.OAuth2Credentials): if credentials.client_id and credentials.client_id != 'null': key_parts.append(credentials.client_id) else: key_parts.append('noclientid') key_parts.append(credentials.refresh_token or 'norefreshtoken') # If a cached credential is targeting provider token URI 'A' for token refresh # requests, then the user changes their boto file or private key file to # target URI 'B', we don't want to treat the cached and the new credential as # interchangeable. This applies for all credentials that store a token URI. if getattr(credentials, 'token_uri', None): key_parts.append(credentials.token_uri) key_parts = [six.ensure_text(part) for part in key_parts] return '-'.join(key_parts) def SetUpJsonCredentialsAndCache(api, logger, credentials=None): """Helper to ensure each GCS API client shares the same credentials.""" api.credentials = (credentials or _CheckAndGetCredentials(logger) or NoOpCredentials()) # Notify the user that impersonation credentials are in effect. if isinstance(api.credentials, ImpersonationCredentials): logger.warn( 'WARNING: This command is using service account impersonation. All ' 'API calls will be executed as [%s].', _GetImpersonateServiceAccount()) # Do not store p12 credentials (Supported by google-auth), as google-auth does not support storing service account credentials. if not isP12Credentials(api.credentials): # Set credential cache so that we don't have to get a new access token for # every call we make. All GCS APIs use the same credentials as the JSON API, # so we use its version in the key for caching access tokens. credential_store_key = (GetCredentialStoreKey(api.credentials, GetGcsJsonApiVersion())) api.credentials.set_store( multiprocess_file_storage.MultiprocessFileStorage( GetCredentialStoreFilename(), credential_store_key)) # The cached entry for this credential often contains more context than what # we can construct from boto config attributes (e.g. for a user credential, # the cached version might also contain a RAPT token and expiry info). # Prefer the cached credential if present. cached_cred = None if not isinstance(api.credentials, NoOpCredentials): # A NoOpCredentials object doesn't actually have a store attribute. cached_cred = api.credentials.store.get() # As of gsutil 4.31, we never use the OAuth2Credentials class for # credentials directly; rather, we use subclasses (user credentials were # the only ones left using it, but they now use # Oauth2WithReauthCredentials). If we detect that a cached credential is # an instance of OAuth2Credentials and not a subclass of it (as might # happen when transitioning to version v4.31+), we don't fetch it from the # cache. This results in our new-style credential being refreshed and # overwriting the old credential cache entry in our credstore. if (cached_cred and type(cached_cred) != oauth2client.client.OAuth2Credentials): api.credentials = cached_cred def _CheckAndGetCredentials(logger): """Returns credentials from the configuration file, if any are present. Args: logger: logging.Logger instance for outputting messages. Returns: OAuth2Credentials object if any valid ones are found, otherwise None. """ configured_cred_types = [] failed_cred_type = None try: if _HasOauth2UserAccountCreds(): configured_cred_types.append(CredTypes.OAUTH2_USER_ACCOUNT) if _HasOauth2ServiceAccountCreds(): configured_cred_types.append(CredTypes.OAUTH2_SERVICE_ACCOUNT) if len(configured_cred_types) > 1: # We only allow one set of configured credentials. Otherwise, we're # choosing one arbitrarily, which can be very confusing to the user # (e.g., if only one is authorized to perform some action) and can # also mask errors. # Because boto merges config files, GCE credentials show up by default # for GCE VMs. We don't want to fail when a user creates a boto file # with their own credentials, so in this case we'll use the OAuth2 # user credentials. raise CommandException( ('You have multiple types of configured credentials (%s), which is ' 'not supported. One common way this happens is if you run gsutil ' 'config to create credentials and later run gcloud auth, and ' 'create a second set of credentials. Your boto config path is: ' '%s. For more help, see "gsutil help creds".') % (configured_cred_types, GetFriendlyConfigFilePaths())) failed_cred_type = CredTypes.OAUTH2_USER_ACCOUNT user_creds = _GetOauth2UserAccountCredentials() failed_cred_type = CredTypes.OAUTH2_SERVICE_ACCOUNT service_account_creds = _GetOauth2ServiceAccountCredentials() failed_cred_type = CredTypes.EXTERNAL_ACCOUNT external_account_creds = _GetExternalAccountCredentials() failed_cred_type = CredTypes.EXTERNAL_ACCOUNT_AUTHORIZED_USER external_account_authorized_user_creds = _GetExternalAccountAuthorizedUserCredentials( ) failed_cred_type = CredTypes.GCE gce_creds = _GetGceCreds() failed_cred_type = CredTypes.DEVSHELL devshell_creds = _GetDevshellCreds() creds = user_creds or service_account_creds or gce_creds or external_account_creds or external_account_authorized_user_creds or devshell_creds # Use one of the above credential types to impersonate, if configured. if _HasImpersonateServiceAccount() and creds: failed_cred_type = CredTypes.IMPERSONATION return _GetImpersonationCredentials(creds, logger) else: return creds except Exception as e: # If we didn't actually try to authenticate because there were multiple # types of configured credentials, don't emit this warning. if failed_cred_type: if logger.isEnabledFor(logging.DEBUG): logger.debug(traceback.format_exc()) # If impersonation fails, show the user the actual error, since we handle # errors in iamcredentials_api. if failed_cred_type == CredTypes.IMPERSONATION: raise e elif system_util.InvokedViaCloudSdk(): logger.warn( 'Your "%s" credentials are invalid. Please run\n' ' $ gcloud auth login', failed_cred_type) else: logger.warn( 'Your "%s" credentials are invalid. For more help, see ' '"gsutil help creds", or re-run the gsutil config command (see ' '"gsutil help config").', failed_cred_type) # If there's any set of configured credentials, we'll fail if they're # invalid, rather than silently falling back to anonymous config (as # boto does). That approach leads to much confusion if users don't # realize their credentials are invalid. raise def _GetProviderTokenUri(): return config.get('OAuth2', 'provider_token_uri', DEFAULT_GOOGLE_OAUTH2_PROVIDER_TOKEN_URI) def _HasOauth2ServiceAccountCreds(): return config.has_option('Credentials', 'gs_service_key_file') def _HasOauth2UserAccountCreds(): return config.has_option('Credentials', 'gs_oauth2_refresh_token') def _HasGceCreds(): return config.has_option('GoogleCompute', 'service_account') def _HasImpersonateServiceAccount(): return _GetImpersonateServiceAccount() not in (None, '') def _GetExternalAccountCredentials(): external_account_filename = config.get('Credentials', 'gs_external_account_file', None) if not external_account_filename: return None return WrappedCredentials.for_external_account(external_account_filename) def _GetExternalAccountAuthorizedUserCredentials(): external_account_authorized_user_filename = config.get( 'Credentials', 'gs_external_account_authorized_user_file', None) if not external_account_authorized_user_filename: return None return WrappedCredentials.for_external_account_authorized_user( external_account_authorized_user_filename) def _GetImpersonateServiceAccount(): return (constants.IMPERSONATE_SERVICE_ACCOUNT or config.get( 'Credentials', 'gs_impersonate_service_account', os.environ.get('CLOUDSDK_AUTH_IMPERSONATE_SERVICE_ACCOUNT'))) def _GetOauth2ServiceAccountCredentials(): """Retrieves OAuth2 service account credentials for a private key file.""" if not _HasOauth2ServiceAccountCreds(): return provider_token_uri = _GetProviderTokenUri() service_client_id = config.get('Credentials', 'gs_service_client_id', '') private_key_filename = config.get('Credentials', 'gs_service_key_file', '') with io.open(private_key_filename, 'rb') as private_key_file: private_key = private_key_file.read() keyfile_is_utf8 = False try: private_key = private_key.decode(UTF8) # P12 keys won't be encoded as UTF8 bytes. keyfile_is_utf8 = True except UnicodeDecodeError: pass if keyfile_is_utf8: try: json_key_dict = json.loads(private_key) except ValueError: raise Exception('Could not parse JSON keyfile "%s" as valid JSON' % private_key_filename) # Key file is in JSON format. for json_entry in ('client_id', 'client_email', 'private_key_id', 'private_key'): if json_entry not in json_key_dict: raise Exception('The JSON private key file at %s ' 'did not contain the required entry: %s' % (private_key_filename, json_entry)) return ServiceAccountCredentials.from_json_keyfile_dict( json_key_dict, scopes=DEFAULT_SCOPES, token_uri=provider_token_uri) else: # Key file is in P12 format. key_file_pass = config.get('Credentials', 'gs_service_key_file_password', GOOGLE_OAUTH2_DEFAULT_FILE_PASSWORD) return CreateP12ServiceAccount(private_key, key_file_pass, scopes=DEFAULT_SCOPES, service_account_email=service_client_id, token_uri=provider_token_uri) def _GetOauth2UserAccountCredentials(): """Retrieves OAuth2 service account credentials for a refresh token.""" if not _HasOauth2UserAccountCreds(): return provider_token_uri = _GetProviderTokenUri() gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) client_id = config.get('OAuth2', 'client_id', os.environ.get('OAUTH2_CLIENT_ID', gsutil_client_id)) client_secret = config.get( 'OAuth2', 'client_secret', os.environ.get('OAUTH2_CLIENT_SECRET', gsutil_client_secret)) # Note that these scopes don't necessarily correspond to the refresh token # being used. This list is is used for obtaining the RAPT in the reauth flow, # to determine which challenges should be used. scopes_for_reauth_challenge = [ constants.Scopes.CLOUD_PLATFORM, constants.Scopes.REAUTH ] return reauth_creds.Oauth2WithReauthCredentials( None, # access_token client_id, client_secret, config.get('Credentials', 'gs_oauth2_refresh_token'), None, # token_expiry provider_token_uri, None, # user_agent scopes=scopes_for_reauth_challenge) def _GetGceCreds(): if not _HasGceCreds(): return try: return credentials_lib.GceAssertionCredentials( service_account_name=config.get('GoogleCompute', 'service_account', 'default'), cache_filename=GetGceCredentialCacheFilename()) except apitools_exceptions.ResourceUnavailableError as e: if 'service account' in str(e) and 'does not exist' in str(e): return None raise def _GetDevshellCreds(): try: return devshell.DevshellCredentials() except devshell.NoDevshellServer: return None except: raise def _GetImpersonationCredentials(credentials, logger): """Retrieves temporary credentials impersonating a service account""" # We don't use impersoned credentials to impersonate. if isinstance(credentials, ImpersonationCredentials): return return ImpersonationCredentials(_GetImpersonateServiceAccount(), [constants.Scopes.CLOUD_PLATFORM], credentials, logger)	20.31 KB	2025-06-25 08:19
📄 gcs_json_media.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Media helper functions and classes for Google Cloud Storage JSON API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import copy import logging import re import socket import types import six from six.moves import http_client from six.moves import urllib from six.moves import cStringIO from apitools.base.py import exceptions as apitools_exceptions from gslib.cloud_api import BadRequestException from gslib.lazy_wrapper import LazyWrapper from gslib.progress_callback import ProgressCallbackWithTimeout from gslib.utils.constants import DEBUGLEVEL_DUMP_REQUESTS from gslib.utils.constants import SSL_TIMEOUT_SEC from gslib.utils.constants import TRANSFER_BUFFER_SIZE from gslib.utils.constants import UTF8 from gslib.utils import text_util import httplib2 from httplib2 import parse_uri if six.PY3: long = int # A regex for matching any series of decimal digits. DECIMAL_REGEX = LazyWrapper(lambda: (re.compile(r'\d+'))) class BytesTransferredContainer(object): """Container class for passing number of bytes transferred to lower layers. For resumed transfers or connection rebuilds in the middle of a transfer, we need to rebuild the connection class with how much we've transferred so far. For uploads, we don't know the total number of bytes uploaded until we've queried the server, but we need to create the connection class to pass to httplib2 before we can query the server. This container object allows us to pass a reference into Upload/DownloadCallbackConnection. """ def __init__(self): self.__bytes_transferred = 0 @property def bytes_transferred(self): return self.__bytes_transferred @bytes_transferred.setter def bytes_transferred(self, value): self.__bytes_transferred = value class UploadCallbackConnectionClassFactory(object): """Creates a class that can override an httplib2 connection. This is used to provide progress callbacks and disable dumping the upload payload during debug statements. It can later be used to provide on-the-fly hash digestion during upload. """ def __init__(self, bytes_uploaded_container, buffer_size=TRANSFER_BUFFER_SIZE, total_size=0, progress_callback=None, logger=None, debug=0): self.bytes_uploaded_container = bytes_uploaded_container self.buffer_size = buffer_size self.total_size = total_size self.progress_callback = progress_callback self.logger = logger self.debug = debug def GetConnectionClass(self): """Returns a connection class that overrides send.""" outer_bytes_uploaded_container = self.bytes_uploaded_container outer_buffer_size = self.buffer_size outer_total_size = self.total_size outer_progress_callback = self.progress_callback outer_logger = self.logger outer_debug = self.debug class UploadCallbackConnection(httplib2.HTTPSConnectionWithTimeout): """Connection class override for uploads.""" bytes_uploaded_container = outer_bytes_uploaded_container # After we instantiate this class, apitools will check with the server # to find out how many bytes remain for a resumable upload. This allows # us to update our progress once based on that number. processed_initial_bytes = False GCS_JSON_BUFFER_SIZE = outer_buffer_size callback_processor = None size = outer_total_size header_encoding = '' header_length = None header_range = None size_modifier = 1.0 def __init__(self, args, kwargs): kwargs['timeout'] = SSL_TIMEOUT_SEC httplib2.HTTPSConnectionWithTimeout.__init__(self, args, *kwargs) # Override httplib.HTTPConnection._send_output for debug logging. # Because the distinction between headers and message body occurs # only in this httplib function, we can only differentiate them here. def _send_output(self, message_body=None, encode_chunked=False): r"""Send the currently buffered request and clear the buffer. Appends an extra \r\n to the buffer. Args: message_body: if specified, this is appended to the request. """ # TODO: Presently, apitools will set http2lib2.debuglevel to 0 # (no prints) or 4 (dump upload payload, httplib prints to stdout). # Refactor to allow our media-handling functions to handle # debuglevel == 4 and print messages to stderr. self._buffer.extend((b'', b'')) if six.PY2: items = self._buffer else: items = [] for item in self._buffer: if isinstance(item, bytes): items.append(item) else: items.append(item.encode(UTF8)) msg = b'\r\n'.join(items) num_metadata_bytes = len(msg) if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug('send: %s' % msg) del self._buffer[:] # If msg and message_body are sent in a single send() call, # it will avoid performance problems caused by the interaction # between delayed ack and the Nagle algorithm. if isinstance(message_body, str): msg += message_body message_body = None self.send(msg, num_metadata_bytes=num_metadata_bytes) if message_body is not None: # message_body was not a string (i.e. it is a file) and # we must run the risk of Nagle self.send(message_body) def putheader(self, header, values): """Overrides HTTPConnection.putheader. Send a request header line to the server. For example: h.putheader('Accept', 'text/html'). This override records the content encoding, length, and range of the payload. For uploads where the content-range difference does not match the content-length, progress printing will under-report progress. These headers are used to calculate a multiplier to correct the progress. For example: the content-length for gzip transport encoded data represents the compressed size of the data while the content-range difference represents the uncompressed size. Dividing the content-range difference by the content-length gives the ratio to multiply the progress by to correctly report the relative progress. Args: header: The header. values: A set of values for the header. """ if header == 'content-encoding': value = ''.join([str(v) for v in values]) self.header_encoding = value if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug( 'send: Using gzip transport encoding for the request.') elif header == 'content-length': try: value = int(''.join([str(v) for v in values])) self.header_length = value except ValueError: pass elif header == 'content-range': try: # There are 3 valid header formats: # '/%d', '%d-%d/', and '%d-%d/%d' value = ''.join([str(v) for v in values]) ranges = DECIMAL_REGEX().findall(value) # If there are 2 or more range values, they will always # correspond to the start and end ranges in the header. if len(ranges) > 1: # Subtract the end position from the start position. self.header_range = (int(ranges[1]) - int(ranges[0])) + 1 except ValueError: pass # If the content header is gzip, and a range and length are set, # update the modifier. if (self.header_encoding == 'gzip' and self.header_length and self.header_range): # Update the modifier self.size_modifier = self.header_range / float(self.header_length) # Reset the headers self.header_encoding = '' self.header_length = None self.header_range = None # Log debug information to catch in tests. if outer_debug == DEBUGLEVEL_DUMP_REQUESTS and outer_logger: outer_logger.debug('send: Setting progress modifier to %s.' % (self.size_modifier)) # Propagate header values. http_client.HTTPSConnection.putheader(self, header, values) def send(self, data, num_metadata_bytes=0): """Overrides HTTPConnection.send. Args: data: string or file-like object (implements read()) of data to send. num_metadata_bytes: number of bytes that consist of metadata (headers, etc.) not representing the data being uploaded. """ if not self.processed_initial_bytes: self.processed_initial_bytes = True if outer_progress_callback: self.callback_processor = ProgressCallbackWithTimeout( outer_total_size, outer_progress_callback) self.callback_processor.Progress( self.bytes_uploaded_container.bytes_transferred) # httplib.HTTPConnection.send accepts either a string or a file-like # object (anything that implements read()). if isinstance(data, six.text_type): full_buffer = cStringIO(data) elif isinstance(data, six.binary_type): full_buffer = six.BytesIO(data) else: full_buffer = data partial_buffer = full_buffer.read(self.GCS_JSON_BUFFER_SIZE) while partial_buffer: if six.PY2: httplib2.HTTPSConnectionWithTimeout.send(self, partial_buffer) else: if isinstance(partial_buffer, bytes): httplib2.HTTPSConnectionWithTimeout.send(self, partial_buffer) else: httplib2.HTTPSConnectionWithTimeout.send( self, partial_buffer.encode(UTF8)) sent_data_bytes = len(partial_buffer) if num_metadata_bytes: if num_metadata_bytes <= sent_data_bytes: sent_data_bytes -= num_metadata_bytes num_metadata_bytes = 0 else: num_metadata_bytes -= sent_data_bytes sent_data_bytes = 0 if self.callback_processor: # Modify the sent data bytes by the size modifier. These are # stored as floats, so the result should be floored. sent_data_bytes = int(sent_data_bytes * self.size_modifier) # TODO: We can't differentiate the multipart upload # metadata in the request body from the actual upload bytes, so we # will actually report slightly more bytes than desired to the # callback handler. Get the number of multipart upload metadata # bytes from apitools and subtract from sent_data_bytes. self.callback_processor.Progress(sent_data_bytes) partial_buffer = full_buffer.read(self.GCS_JSON_BUFFER_SIZE) return UploadCallbackConnection def WrapUploadHttpRequest(upload_http): """Wraps upload_http so we only use our custom connection_type on PUTs. POSTs are used to refresh oauth tokens, and we don't want to process the data sent in those requests. Args: upload_http: httplib2.Http instance to wrap """ request_orig = upload_http.request def NewRequest(uri, method='GET', body=None, headers=None, redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): if method == 'PUT' or method == 'POST': override_connection_type = connection_type else: override_connection_type = None return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=override_connection_type) # Replace the request method with our own closure. upload_http.request = NewRequest class DownloadCallbackConnectionClassFactory(object): """Creates a class that can override an httplib2 connection. This is used to provide progress callbacks, disable dumping the download payload during debug statements, and provide on-the-fly hash digestion during download. On-the-fly digestion is particularly important because httplib2 will decompress gzipped content on-the-fly, thus this class provides our only opportunity to calculate the correct hash for an object that has a gzip hash in the cloud. """ def __init__(self, bytes_downloaded_container, buffer_size=TRANSFER_BUFFER_SIZE, total_size=0, progress_callback=None, digesters=None): self.buffer_size = buffer_size self.total_size = total_size self.progress_callback = progress_callback self.digesters = digesters self.bytes_downloaded_container = bytes_downloaded_container def GetConnectionClass(self): """Returns a connection class that overrides getresponse.""" class DownloadCallbackConnection(httplib2.HTTPSConnectionWithTimeout): """Connection class override for downloads.""" outer_total_size = self.total_size outer_digesters = self.digesters outer_progress_callback = self.progress_callback outer_bytes_downloaded_container = self.bytes_downloaded_container processed_initial_bytes = False callback_processor = None def __init__(self, args, kwargs): kwargs['timeout'] = SSL_TIMEOUT_SEC httplib2.HTTPSConnectionWithTimeout.__init__(self, args, *kwargs) def getresponse(self, buffering=False): """Wraps an HTTPResponse to perform callbacks and hashing. In this function, self is a DownloadCallbackConnection. Args: buffering: Unused. This function uses a local buffer. Returns: HTTPResponse object with wrapped read function. """ orig_response = http_client.HTTPConnection.getresponse(self) if orig_response.status not in (http_client.OK, http_client.PARTIAL_CONTENT): return orig_response orig_read_func = orig_response.read def read(amt=None): # pylint: disable=invalid-name """Overrides HTTPConnection.getresponse.read. This function only supports reads of TRANSFER_BUFFER_SIZE or smaller. Args: amt: Integer n where 0 < n <= TRANSFER_BUFFER_SIZE. This is a keyword argument to match the read function it overrides, but it is required. Returns: Data read from HTTPConnection. """ if not amt or amt > TRANSFER_BUFFER_SIZE: raise BadRequestException( 'Invalid HTTP read size %s during download, expected %s.' % (amt, TRANSFER_BUFFER_SIZE)) else: amt = amt or TRANSFER_BUFFER_SIZE if not self.processed_initial_bytes: self.processed_initial_bytes = True if self.outer_progress_callback: self.callback_processor = ProgressCallbackWithTimeout( self.outer_total_size, self.outer_progress_callback) self.callback_processor.Progress( self.outer_bytes_downloaded_container.bytes_transferred) data = orig_read_func(amt) read_length = len(data) if self.callback_processor: self.callback_processor.Progress(read_length) if self.outer_digesters: for alg in self.outer_digesters: self.outer_digesters[alg].update(data) return data orig_response.read = read return orig_response return DownloadCallbackConnection def WrapDownloadHttpRequest(download_http): """Overrides download request functions for an httplib2.Http object. Args: download_http: httplib2.Http.object to wrap / override. Returns: Wrapped / overridden httplib2.Http object. """ # httplib2 has a bug (https://github.com/httplib2/httplib2/issues/75) where # custom connection_type is not respected after redirects. This function is # copied from httplib2 and overrides the request function so that the # connection_type is properly passed through (everything here should be # identical to the _request method in httplib2, with the exception of the line # below marked by the "BUGFIX" comment). # pylint: disable=protected-access,g-inconsistent-quotes,unused-variable # pylint: disable=g-equals-none,g-doc-return-or-yield # pylint: disable=g-short-docstring-punctuation,g-doc-args # pylint: disable=too-many-statements # yapf: disable def OverrideRequest(self, conn, host, absolute_uri, request_uri, method, body, headers, redirections, cachekey): """Do the actual request using the connection object. Also follow one level of redirects if necessary. """ auths = ([(auth.depth(request_uri), auth) for auth in self.authorizations if auth.inscope(host, request_uri)]) auth = auths and sorted(auths)[0][1] or None if auth: auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) if auth: if auth.response(response, body): auth.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) response._stale_digest = 1 if response.status == 401: for authorization in self._auth_from_challenge( host, request_uri, headers, response, content): authorization.request(method, request_uri, headers, body) (response, content) = self._conn_request(conn, request_uri, method, body, headers) if response.status != 401: self.authorizations.append(authorization) authorization.response(response, body) break if (self.follow_all_redirects or (method in ["GET", "HEAD"]) or response.status == 303): if self.follow_redirects and response.status in [300, 301, 302, 303, 307]: # Pick out the location header and basically start from the beginning # remembering first to strip the ETag header and decrement our 'depth' if redirections: if 'location' not in response and response.status != 300: raise httplib2.RedirectMissingLocation( "Redirected but the response is missing a Location: header.", response, content) # Fix-up relative redirects (which violate an RFC 2616 MUST) if 'location' in response: location = response['location'] (scheme, authority, path, query, fragment) = parse_uri(location) if authority is None: response['location'] = urllib.parse.urljoin(absolute_uri, location) if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if 'content-location' not in response: response['content-location'] = absolute_uri httplib2._updateCache(headers, response, content, self.cache, cachekey) if 'if-none-match' in headers: del headers['if-none-match'] if 'if-modified-since' in headers: del headers['if-modified-since'] if ('authorization' in headers and not self.forward_authorization_headers): del headers['authorization'] if 'location' in response: location = response['location'] old_response = copy.deepcopy(response) if 'content-location' not in old_response: old_response['content-location'] = absolute_uri redirect_method = method if response.status in [302, 303]: redirect_method = "GET" body = None (response, content) = self.request( location, redirect_method, body=body, headers=headers, redirections=redirections-1, # BUGFIX (see comments at the top of this function): connection_type=conn.__class__) response.previous = old_response else: raise httplib2.RedirectLimit( "Redirected more times than redirection_limit allows.", response, content) elif response.status in [200, 203] and method in ["GET", "HEAD"]: # Don't cache 206's since we aren't going to handle byte range # requests if 'content-location' in response: response['content-location'] = absolute_uri httplib2._updateCache(headers, response, content, self.cache, cachekey) return (response, content) # Wrap download_http so we do not use our custom connection_type # on POSTS, which are used to refresh oauth tokens. We don't want to # process the data received in those requests. request_orig = download_http.request def NewRequest(uri, method='GET', body=None, headers=None, redirections=httplib2.DEFAULT_MAX_REDIRECTS, connection_type=None): if method == 'POST': return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=None) else: return request_orig(uri, method=method, body=body, headers=headers, redirections=redirections, connection_type=connection_type) # Replace the request methods with our own closures. download_http._request = types.MethodType(OverrideRequest, download_http) download_http.request = NewRequest return download_http class HttpWithNoRetries(httplib2.Http): """httplib2.Http variant that does not retry. httplib2 automatically retries requests according to httplib2.RETRIES, but in certain cases httplib2 ignores the RETRIES value and forces a retry. Because httplib2 does not handle the case where the underlying request body is a stream, a retry may cause a non-idempotent write as the stream is partially consumed and not reset before the retry occurs. Here we override _conn_request to disable retries unequivocally, so that uploads may be retried at higher layers that properly handle stream request bodies. """ def _conn_request(self, conn, request_uri, method, body, headers): # pylint: disable=too-many-statements try: if hasattr(conn, 'sock') and conn.sock is None: conn.connect() conn.request(method, request_uri, body, headers) except socket.timeout: raise except socket.gaierror: conn.close() raise httplib2.ServerNotFoundError('Unable to find the server at %s' % conn.host) except httplib2.ssl.SSLError: conn.close() raise except socket.error as e: err = 0 if hasattr(e, 'args'): err = getattr(e, 'args')[0] else: err = e.errno if err == httplib2.errno.ECONNREFUSED: # Connection refused raise except http_client.HTTPException: conn.close() raise try: response = conn.getresponse() except (socket.error, http_client.HTTPException): conn.close() raise else: content = '' if method == 'HEAD': conn.close() else: content = response.read() response = httplib2.Response(response) if method != 'HEAD': # pylint: disable=protected-access content = httplib2._decompressContent(response, content) return (response, content) class HttpWithDownloadStream(httplib2.Http): """httplib2.Http variant that only pushes bytes through a stream. httplib2 handles media by storing entire chunks of responses in memory, which is undesirable particularly when multiple instances are used during multi-threaded/multi-process copy. This class copies and then overrides some httplib2 functions to use a streaming copy approach that uses small memory buffers. Also disables httplib2 retries (for reasons stated in the HttpWithNoRetries class doc). """ def __init__(self, args, *kwds): self._stream = None self._logger = logging.getLogger() super(HttpWithDownloadStream, self).__init__(args, **kwds) @property def stream(self): return self._stream @stream.setter def stream(self, value): self._stream = value # pylint: disable=too-many-statements def _conn_request(self, conn, request_uri, method, body, headers): try: if hasattr(conn, 'sock') and conn.sock is None: conn.connect() conn.request(method, request_uri, body, headers) except socket.timeout: raise except socket.gaierror: conn.close() raise httplib2.ServerNotFoundError('Unable to find the server at %s' % conn.host) except httplib2.ssl.SSLError: conn.close() raise except socket.error as e: err = 0 if hasattr(e, 'args'): err = getattr(e, 'args')[0] else: err = e.errno if err == httplib2.errno.ECONNREFUSED: # Connection refused raise except http_client.HTTPException: # Just because the server closed the connection doesn't apparently mean # that the server didn't send a response. conn.close() raise try: response = conn.getresponse() except (socket.error, http_client.HTTPException) as e: conn.close() raise else: content = '' if method == 'HEAD': conn.close() response = httplib2.Response(response) elif method == 'GET' and response.status in (http_client.OK, http_client.PARTIAL_CONTENT): content_length = None if hasattr(response, 'msg'): content_length = response.getheader('content-length') http_stream = response bytes_read = 0 while True: new_data = http_stream.read(TRANSFER_BUFFER_SIZE) if new_data: if self.stream is None: raise apitools_exceptions.InvalidUserInputError( 'Cannot exercise HttpWithDownloadStream with no stream') text_util.write_to_fd(self.stream, new_data) bytes_read += len(new_data) else: break if (content_length is not None and long(bytes_read) != long(content_length)): # The input stream terminated before we were able to read the # entire contents, possibly due to a network condition. Set # content-length to indicate how many bytes we actually read. self._logger.log( logging.DEBUG, 'Only got %s bytes out of content-length %s ' 'for request URI %s. Resetting content-length to match ' 'bytes read.', bytes_read, content_length, request_uri) # Failing to delete existing headers before setting new values results # in the header being set twice, see https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__. # This trips apitools up when executing a retry, so the line below is # essential: del response.msg['content-length'] response.msg['content-length'] = str(bytes_read) response = httplib2.Response(response) else: # We fall back to the current httplib2 behavior if we're # not processing download bytes, e.g., it's a redirect, an # oauth2client POST to refresh an access token, or any HTTP # status code that doesn't include object content. content = response.read() response = httplib2.Response(response) # pylint: disable=protected-access content = httplib2._decompressContent(response, content) return (response, content) # pylint: enable=too-many-statements	28.5 KB	2025-06-25 08:19
📄 help_provider.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Module defining help types and providers for gsutil commands.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections from gslib.exception import CommandException ALL_HELP_TYPES = ['command_help', 'additional_help'] # Constants enforced by SanityCheck # Mainly here for help output formatting purposes, can be changed if necessary. MAX_HELP_NAME_LEN = 17 MIN_ONE_LINE_SUMMARY_LEN = 10 MAX_ONE_LINE_SUMMARY_LEN = 80 - MAX_HELP_NAME_LEN DESCRIPTION_PREFIX = """ <B>DESCRIPTION</B>""" SYNOPSIS_PREFIX = """ <B>SYNOPSIS</B>""" class HelpProvider(object): """Interface for providing help.""" # Each subclass of HelpProvider define a property named 'help_spec' that is # an instance of the following class. HelpSpec = collections.namedtuple( 'HelpSpec', [ # Name of command or auxiliary help info for which this help applies. 'help_name', # List of help name aliases. 'help_name_aliases', # Type of help. 'help_type', # One line summary of this help. 'help_one_line_summary', # The full help text. 'help_text', # Help text for subcommands of the command's help being specified. 'subcommand_help_text', ]) # Each subclass must override this with an instance of HelpSpec. help_spec = None # This is a static helper instead of a class method because the help loader # (gslib.commands.help._LoadHelpMaps()) operates on classes not instances. def SanityCheck(help_provider, help_name_map): """Helper for checking that a HelpProvider has minimally adequate content.""" # Sanity check the content. help_name_len = len(help_provider.help_spec.help_name) assert (help_name_len > 1 and help_name_len < MAX_HELP_NAME_LEN ), 'The help name "{text}" must be less then {max}'.format( text=help_provider.help_spec.help_name, max=MAX_HELP_NAME_LEN) for hna in help_provider.help_spec.help_name_aliases: assert hna one_line_summary_len = len(help_provider.help_spec.help_one_line_summary) assert (one_line_summary_len >= MIN_ONE_LINE_SUMMARY_LEN), ( 'The one line summary "{text}" with a length of {length} must be ' + 'more then {min} characters').format( text=help_provider.help_spec.help_one_line_summary, length=one_line_summary_len, min=MIN_ONE_LINE_SUMMARY_LEN) assert (one_line_summary_len <= MAX_ONE_LINE_SUMMARY_LEN), ( 'The one line summary "{text}" with a length of {length} must be ' + 'less then {max} characters').format( text=help_provider.help_spec.help_one_line_summary, length=one_line_summary_len, max=MAX_ONE_LINE_SUMMARY_LEN) assert len(help_provider.help_spec.help_text ) > 10, 'The length of "{text}" must be less then 10'.format( text=help_provider.help_spec.help_text) # Ensure there are no dupe help names or aliases across commands. name_check_list = [help_provider.help_spec.help_name] name_check_list.extend(help_provider.help_spec.help_name_aliases) for name_or_alias in name_check_list: if name_or_alias in help_name_map: raise CommandException( 'Duplicate help name/alias "%s" found while loading help from %s. ' 'That name/alias was already taken by %s' % (name_or_alias, help_provider.__module__, help_name_map[name_or_alias].__module__)) def CreateHelpText(synopsis, description): """Helper for adding help text headers given synopsis and description.""" return SYNOPSIS_PREFIX + synopsis + DESCRIPTION_PREFIX + description	4.26 KB	2025-06-25 08:19
📄 iamcredentials_api.py # -- coding: utf-8 -- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.iamcredentials_apitools import iamcredentials_v1_client as apitools_client from gslib.third_party.iamcredentials_apitools import iamcredentials_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class IamcredentailsApi(object): """Wraps calls to the Cloud IAM Credentials v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud IAM Credentials. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud IAM debug: Debug level for the API implementation (0..3). """ super(IamcredentailsApi, self).__init__() self.logger = logger self.credentials = credentials self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' # The original config option had a typo; supporting it as a fall-back # for legacy reasons. legacy_typo = 'gs_iamcredentails_host' self.host_base = config.get( 'Credentials', 'gs_iamcredentials_host', config.get('Credentials', legacy_typo, 'iamcredentials.googleapis.com')) gs_iamcred_port = config.get('Credentials', 'gs_iamcredentails_port', None) self.host_port = (':' + gs_iamcred_port) if gs_iamcred_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.IamcredentialsV1( url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam( 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def SignBlob(self, service_account_id, message): """Sign the blob using iamcredentials.SignBlob API.""" name = 'projects/-/serviceAccounts/%s' % service_account_id sign_blob_request = apitools_messages.SignBlobRequest(payload=message) request = ( apitools_messages.IamcredentialsProjectsServiceAccountsSignBlobRequest( name=name, signBlobRequest=sign_blob_request)) try: return self.api_client.projects_serviceAccounts.SignBlob(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, service_account_id) def GenerateAccessToken(self, service_account_id, scopes): """Generates an access token for the given service account.""" name = 'projects/-/serviceAccounts/%s' % service_account_id generate_access_token_request = apitools_messages.GenerateAccessTokenRequest( scope=scopes) request = (apitools_messages. IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest( name=name, generateAccessTokenRequest=generate_access_token_request)) try: return self.api_client.projects_serviceAccounts.GenerateAccessToken( request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, service_account_id) def _TranslateExceptionAndRaise(self, e, service_account_id=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. service_account_id: Optional service account in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, service_account_id=service_account_id) if translated_exception: raise translated_exception else: raise def _TranslateApitoolsException(self, e, service_account_id=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. service_account_id: Optional service account ID that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: # Messaging for when the the originating credentials don't have access # to impersonate a service account. if 'The caller does not have permission' in str(e): return AccessDeniedException( 'Service account impersonation failed. Please go to the Google ' 'Cloud Platform Console (https://cloud.google.com/console), ' 'select IAM & admin, then Service Accounts, and grant your ' 'originating account the Service Account Token Creator role on ' 'the target service account.') # The server's errors message when IAM Credentials API aren't enabled # are pretty great so we just display them. if 'IAM Service Account Credentials API has not been used' in str(e): return AccessDeniedException(message) if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs & services, and enable the Google Cloud ' 'IAM Credentials API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message or service_account_id, status=e.status_code) elif e.status_code == 404: return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and service_account_id: return ServiceException('The key %s already exists.' % service_account_id, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code) def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass	11.72 KB	2025-06-25 08:19
📄 impersonation_credentials.py # -- coding: utf-8 -- # Copyright 2019 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Implementation of credentials that refreshes using the iamcredentials API.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import datetime from oauth2client import client from gslib.iamcredentials_api import IamcredentailsApi class ImpersonationCredentials(client.OAuth2Credentials): _EXPIRY_FORMAT = '%Y-%m-%dT%H:%M:%SZ' def __init__(self, service_account_id, scopes, credentials, logger): self._service_account_id = service_account_id self.api = IamcredentailsApi(logger, credentials) response = self.api.GenerateAccessToken(service_account_id, scopes) self.access_token = response.accessToken self.token_expiry = self._ConvertExpiryTime(response.expireTime) super(ImpersonationCredentials, self).__init__(self.access_token, None, None, None, self.token_expiry, None, None, scopes=scopes) @property def service_account_id(self): return self._service_account_id def _refresh(self, http): # client.Oauth2Credentials converts scopes into a set, so we need to convert # back to a list before making the API request. response = self.api.GenerateAccessToken(self._service_account_id, list(self.scopes)) self.access_token = response.accessToken self.token_expiry = self._ConvertExpiryTime(response.expireTime) def _ConvertExpiryTime(self, value): return datetime.datetime.strptime(value, ImpersonationCredentials._EXPIRY_FORMAT)	2.51 KB	2025-06-25 08:19
📄 kms_api.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.kms_apitools import cloudkms_v1_client as apitools_client from gslib.third_party.kms_apitools import cloudkms_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class KmsApi(object): """Wraps calls to the Cloud KMS v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud KMS. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Cloud KMS debug: Debug level for the API implementation (0..3). """ super(KmsApi, self).__init__() self.logger = logger self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' self.host_base = config.get('Credentials', 'gs_kms_host', 'cloudkms.googleapis.com') gs_kms_port = config.get('Credentials', 'gs_kms_port', None) self.host_port = (':' + gs_kms_port) if gs_kms_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.CloudkmsV1(url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam( 'key', u'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetKeyIamPolicy(self, key_name): request = (apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysGetIamPolicyRequest( resource=key_name)) try: return (self.api_client.projects_locations_keyRings_cryptoKeys. GetIamPolicy(request)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, key_name=key_name) def SetKeyIamPolicy(self, key_name, policy): policy_request = apitools_messages.SetIamPolicyRequest(policy=policy) request = (apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysSetIamPolicyRequest( resource=key_name, setIamPolicyRequest=policy_request)) try: return (self.api_client.projects_locations_keyRings_cryptoKeys. SetIamPolicy(request)) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, key_name=key_name) def CreateKeyRing(self, project, keyring_name, location='global'): """Attempts to create the specified keyRing. Args: project: (str) The project id in which to create the keyRing and key. keyring_name: (str) The name of the keyRing, e.g. my-keyring. Note that this must be unique within the location. location: (str) The location in which to create the keyRing. Defaults to 'global'. Returns: (str) The fully-qualified name of the keyRing, e.g.: projects/my-project/locations/global/keyRings/my-keyring Raises: Translated CloudApi exception if we were unable to create the keyRing. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. """ keyring_msg = apitools_messages.KeyRing( name='projects/%s/locations/%s/keyRings/%s' % (project, location, keyring_name)) keyring_create_request = ( apitools_messages.CloudkmsProjectsLocationsKeyRingsCreateRequest( keyRing=keyring_msg, keyRingId=keyring_name, parent='projects/%s/locations/%s' % (project, location))) try: self.api_client.projects_locations_keyRings.Create(keyring_create_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if e.status_code != 409: raise return 'projects/%s/locations/%s/keyRings/%s' % (project, location, keyring_name) def CreateCryptoKey(self, keyring_fqn, key_name): """Attempts to create the specified cryptoKey. Args: keyring_fqn: (str) The fully-qualified name of the keyRing, e.g. projects/my-project/locations/global/keyRings/my-keyring. key_name: (str) The name of the desired key, e.g. my-key. Note that this must be unique within the keyRing. Returns: (str) The fully-qualified name of the cryptoKey, e.g.: projects/my-project/locations/global/keyRings/my-keyring/cryptoKeys/my-key Raises: Translated CloudApi exception if we were unable to create the cryptoKey. Note that in the event of a 409 status code (resource already exists) when attempting creation, we continue and treat this as a success. """ cryptokey_msg = apitools_messages.CryptoKey(purpose=( apitools_messages.CryptoKey.PurposeValueValuesEnum.ENCRYPT_DECRYPT)) cryptokey_create_request = ( apitools_messages. CloudkmsProjectsLocationsKeyRingsCryptoKeysCreateRequest( cryptoKey=cryptokey_msg, cryptoKeyId=key_name, parent=keyring_fqn)) try: self.api_client.projects_locations_keyRings_cryptoKeys.Create( cryptokey_create_request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: if e.status_code != 409: raise return '%s/cryptoKeys/%s' % (keyring_fqn.rstrip('/'), key_name) def _TranslateExceptionAndRaise(self, e, key_name=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. key_name: Optional key name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException(e, key_name=key_name) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsException(self, e, key_name=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. key_name: Optional key name in request that caused the exception. Returns: CloudStorageApiServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs & services, and enable the Google Cloud ' 'KMS API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message or key_name, status=e.status_code) elif e.status_code == 404: return NotFoundException(message or e.message, status=e.status_code) elif e.status_code == 409 and key_name: return ServiceException('The key %s already exists.' % key_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code)	13.24 KB	2025-06-25 08:19
📄 lazy_wrapper.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a LazyWrapper class to defer loading a value until it's needed.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class LazyWrapper(object): """Wrapper for lazily instantiated objects.""" def __init__(self, func): """The init method for LazyWrapper. Args: func: A function (lambda or otherwise) to lazily evaluate. """ self._func = func def __int__(self): try: return int(self._value) except AttributeError: self._value = self._func() return int(self._value) def __eq__(self, other): try: return self._value == other except AttributeError: self._value = self._func() return self._value == other def __repr__(self): try: return str(self._value) except AttributeError: self._value = self._func() return str(self._value) def __str__(self): try: return str(self._value) except AttributeError: self._value = self._func() return str(self._value) def __call__(self): """The call method for a LazyWrapper object.""" try: return self._value except AttributeError: self._value = self._func() return self._value def __len__(self): """The len method for a LazyWrapper object.""" try: return len(self._value) except AttributeError: self.__call__() return len(self._value) def __iter__(self): """The iter method for a LazyWrapper object.""" try: return self._value.__iter__() except AttributeError: self.__call__() return self._value.__iter__()	2.26 KB	2025-06-25 08:19
📄 metrics.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Static data and helper functions for collecting user data.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import atexit from collections import defaultdict from functools import wraps import logging import os import pickle import platform import re import socket import subprocess import sys import tempfile import textwrap import time import uuid import six from six.moves import input from six.moves import urllib import boto from gslib import VERSION from gslib.metrics_tuple import Metric from gslib.utils import system_util from gslib.utils.unit_util import CalculateThroughput from gslib.utils.unit_util import HumanReadableToBytes _GA_ENDPOINT = 'https://ssl.google-analytics.com/collect' _GA_TID = 'UA-36037335-16' _GA_TID_TESTING = 'UA-36037335-17' _GA_COMMANDS_CATEGORY = 'Command' _GA_ERRORRETRY_CATEGORY = 'RetryableError' _GA_ERRORFATAL_CATEGORY = 'FatalError' _GA_PERFSUM_CATEGORY = 'PerformanceSummary' # This approximates whether the host is on a Google corporate network, # but since users choose their own hostnames, it is not guaranteed to be # accurate. _GOOGLE_CORP_HOST_RE = re.compile(r'.google\.com$') _UUID_FILE_PATH = os.path.expanduser(os.path.join('~', '.gsutil/analytics-uuid')) # If this string is written to analytics-uuid, that means that the user said # 'no' to analytics, and it should thus be disabled. _DISABLED_TEXT = 'DISABLED' # Analytics collection uses the environment variable 'GSUTIL_TEST_ANALYTICS'. # - A value of '1' completely disables analytics collection. This is used during # non-analytics tests to avoid reporting analytics events during normal # testing. # - A value of '2' sets testing parameters and prevents the metrics_reporter.py # subprocess from writing to the metrics log file. This is used during # analytics tests in integration test subprocesses to disable reporting to GA # and avoid overwriting the metrics log file. In this scenario, the main # process’s root logger is enabled for debug output, so we still want # collection to occur so we can read metrics log messages from stderr. # - Any other value sets default behavior. # Map from descriptive labels to the key labels that GA recognizes. _GA_LABEL_MAP = { 'Event Category': 'ec', 'Event Action': 'ea', 'Event Label': 'el', 'Event Value': 'ev', 'Command Name': 'cd1', 'Global Options': 'cd2', 'Command-Level Options': 'cd3', 'Config': 'cd4', 'Command Alias': 'cd5', 'Fatal Error': 'cd6', 'Parallelism Strategy': 'cd7', 'Source URL Type': 'cd8', 'Provider Types': 'cd9', 'Timestamp': 'cd10', 'Execution Time': 'cm1', 'Retryable Errors': 'cm2', 'Is Google Corp User': 'cm3', 'Num Processes': 'cm4', 'Num Threads': 'cm5', 'Number of Files/Objects Transferred': 'cm6', 'Size of Files/Objects Transferred': 'cm7', 'Average Overall Throughput': 'cm8', 'Num Retryable Service Errors': 'cm9', 'Num Retryable Network Errors': 'cm10', 'Thread Idle Time Percent': 'cm11', 'Slowest Thread Throughput': 'cm12', 'Fastest Thread Throughput': 'cm13', 'Disk I/O Time': 'cm14', } class MetricsCollector(object): """A singleton class to handle metrics reporting to Google Analytics (GA). This class is not thread or process-safe, and logging directly to the MetricsCollector instance can only be done by a single thread. """ def __init__(self, ga_tid=_GA_TID, endpoint=_GA_ENDPOINT): """Initialize a new MetricsCollector. This should only be invoked through the GetCollector or StartTestCollector functions. Args: ga_tid: The Google Analytics tracking ID to use for metrics collection. endpoint: The URL to send requests to. """ self.start_time = _GetTimeInMillis() self.endpoint = endpoint self.logger = logging.getLogger() # Calculate parameters to send with every GA event, stored as ga_params. cid = MetricsCollector._GetCID() # gsutil developers should set this config value to true in order to hit the # testing GA property rather than the production property. use_test_property = boto.config.getbool('GSUtil', 'use_test_GA_property') if use_test_property: ga_tid = _GA_TID_TESTING # A string of non-PII config values. config_values = self._ValidateAndGetConfigValues() # Approximate if this is a Google corporate user. is_corp_user = 0 if _GOOGLE_CORP_HOST_RE.match(socket.gethostname()): is_corp_user = 1 self.ga_params = { 'v': '1', 'tid': ga_tid, 'cid': cid, 't': 'event', _GA_LABEL_MAP['Config']: config_values, _GA_LABEL_MAP['Is Google Corp User']: is_corp_user, } # Used by Google Analytics to track user OS. self.user_agent = '{system}/{release}'.format(system=platform.system(), release=platform.release()) # A list of collected, unsent Metrics. This list is currently bounded by # the number of retryable error types, and should not grow too large so that # we stay well within memory constraints. self._metrics = [] # Store a count of the number of each type of retryable error. self.retryable_errors = defaultdict(int) # This will be set to keep track of information necessary for a # PerformanceSummary if the cp or rsync commands are run. self.perf_sum_params = None _instance = None # Whether analytics collection is disabled or not. _disabled_cache = None def _ValidateAndGetConfigValues(self): """Parses the user's config file to aggregate non-PII config values. Returns: A comma-delimited string of config values explicitly set by the user in key:value pairs, sorted alphabetically by key. """ config_values = [] # If a user has an invalid config value set, we will mark it as such with # this value. If a user did not enter a value, we will not report it. invalid_value_string = 'INVALID' def GetAndValidateConfigValue(section, category, validation_fn): try: config_value = boto.config.get_value(section, category) if config_value and validation_fn(config_value): config_values.append((category, config_value)) # If the user entered a non-valid config value, store it as invalid. elif config_value: config_values.append((category, invalid_value_string)) # This function gets called during initialization of the MetricsCollector. # If any of the logic fails, we do not want to hinder the gsutil command # being run, and thus ignore any exceptions. except: # pylint: disable=bare-except config_values.append((category, invalid_value_string)) # Validate boolean values. for section, bool_category in (('Boto', 'https_validate_certificates'), ('GSUtil', 'disable_analytics_prompt'), ('GSUtil', 'use_magicfile'), ('GSUtil', 'tab_completion_time_logs')): GetAndValidateConfigValue(section=section, category=bool_category, validation_fn=lambda val: str(val).lower() in ('true', 'false')) # Define a threshold for some config values which should be reasonably low. small_int_threshold = 2000 # Validate small integers. for section, small_int_category in ( ('Boto', 'debug'), ('Boto', 'http_socket_timeout'), ('Boto', 'num_retries'), ('Boto', 'max_retry_delay'), ('GSUtil', 'default_api_version'), ('GSUtil', 'sliced_object_download_max_components'), ('GSUtil', 'parallel_process_count'), ('GSUtil', 'parallel_thread_count'), ('GSUtil', 'software_update_check_period'), ('GSUtil', 'tab_completion_timeout'), ('OAuth2', 'oauth2_refresh_retries'), ): GetAndValidateConfigValue(section=section, category=small_int_category, validation_fn=lambda val: str(val).isdigit() and int(val) < small_int_threshold) # Validate large integers. for section, large_int_category in (('GSUtil', 'resumable_threshold'), ('GSUtil', 'rsync_buffer_lines'), ('GSUtil', 'task_estimation_threshold')): GetAndValidateConfigValue(section=section, category=large_int_category, validation_fn=lambda val: str(val).isdigit()) # Validate data sizes. for section, data_size_category in ( ('GSUtil', 'parallel_composite_upload_component_size'), ('GSUtil', 'parallel_composite_upload_threshold'), ('GSUtil', 'sliced_object_download_component_size'), ('GSUtil', 'sliced_object_download_threshold'), ): config_value = boto.config.get_value(section, data_size_category) if config_value: try: size_in_bytes = HumanReadableToBytes(config_value) config_values.append((data_size_category, size_in_bytes)) except ValueError: config_values.append((data_size_category, invalid_value_string)) # Validate specific options. # pylint: disable=g-long-lambda GetAndValidateConfigValue( section='GSUtil', category='check_hashes', validation_fn=lambda val: val in ('if_fast_else_fail', 'if_fast_else_skip', 'always', 'never')) # pylint: enable=g-long-lambda GetAndValidateConfigValue( section='GSUtil', category='content_language', validation_fn=lambda val: val.isalpha() and len(val) <= 3) GetAndValidateConfigValue( section='GSUtil', category='json_api_version', validation_fn=lambda val: val[0].lower() == 'v' and val[1:].isdigit()) GetAndValidateConfigValue(section='GSUtil', category='prefer_api', validation_fn=lambda val: val in ('json', 'xml')) GetAndValidateConfigValue(section='OAuth2', category='token_cache', validation_fn=lambda val: val in ('file_system', 'in_memory')) return ','.join( sorted([ '{0}:{1}'.format(config[0], config[1]) for config in config_values ])) @staticmethod def GetCollector(ga_tid=_GA_TID): """Returns the singleton MetricsCollector instance or None if disabled.""" if MetricsCollector.IsDisabled(): return None if not MetricsCollector._instance: MetricsCollector._instance = MetricsCollector(ga_tid) return MetricsCollector._instance @staticmethod def IsDisabled(): """Returns whether metrics collection should be disabled.""" if MetricsCollector._disabled_cache is None: MetricsCollector._CheckAndSetDisabledCache() return MetricsCollector._disabled_cache @classmethod def _CheckAndSetDisabledCache(cls): """Sets _disabled_cache based on user opt-in or out.""" # Disable collection for a test case where no metrics should be collected. if os.environ.get('GSUTIL_TEST_ANALYTICS') == '1': cls._disabled_cache = True # Enable test collector for a subprocess integration test case where we # check the log output, which requires a test collector. elif os.environ.get('GSUTIL_TEST_ANALYTICS') == '2': cls._disabled_cache = False cls.StartTestCollector() # Non-testing cases involve checking for shim usage, the cloud SDK wrapper # and the analytics uuid file. elif boto.config.getbool('GSUtil', 'use_gcloud_storage', False): cls._disabled_cache = True elif system_util.InvokedViaCloudSdk(): cls._disabled_cache = not os.environ.get('GA_CID') elif os.path.exists(_UUID_FILE_PATH): with open(_UUID_FILE_PATH) as f: cls._disabled_cache = (f.read() == _DISABLED_TEXT) else: cls._disabled_cache = True @classmethod def StartTestCollector(cls, endpoint='https://example.com', user_agent='user-agent-007', ga_params=None): """Reset the singleton MetricsCollector with testing parameters. Should only be used for tests, where we want to change the default parameters. Args: endpoint: str, URL to post to user_agent: str, User-Agent string for header. ga_params: A list of two-dimensional string tuples to send as parameters. """ # Re-enable analytics for the duration of the testing. if cls.IsDisabled(): os.environ['GSUTIL_TEST_ANALYTICS'] = '0' cls._disabled_cache = False cls._instance = cls(_GA_TID_TESTING, endpoint) if ga_params is None: ga_params = {'a': 'b', 'c': 'd'} cls._instance.ga_params = ga_params cls._instance.user_agent = user_agent if os.environ['GSUTIL_TEST_ANALYTICS'] != '2': cls._instance.start_time = 0 @classmethod def StopTestCollector(cls, original_instance=None): """Reset the MetricsCollector with default parameters after testing. Args: original_instance: The original instance of the MetricsCollector so we can set the collector back to its original state. """ os.environ['GSUTIL_TEST_ANALYTICS'] = '1' cls._disabled_cache = None cls._instance = original_instance @staticmethod def _GetCID(): """Gets the client id from the UUID file or the SDK opt-in, or returns None. Returns: str, The hex string of the client id. """ if os.path.exists(_UUID_FILE_PATH): with open(_UUID_FILE_PATH) as f: cid = f.read() if cid: return cid # Returns CID from SDK. This value will be None if there is no opt-in from # the SDK. return os.environ.get('GA_CID') def ExtendGAParams(self, new_params): """Extends self.ga_params to include new parameters. This is only used to record parameters that are sent with every event type, such as global and command-level options. Args: new_params: A dictionary of key-value parameters to send. """ self.ga_params.update(new_params) def GetGAParam(self, param_name): """Convenience function for getting a ga_param of the collector. Args: param_name: The descriptive name of the param (e.g. 'Command Name'). Must be a key in _GA_LABEL_MAP. Returns: The GA parameter specified, or None. """ return self.ga_params.get(_GA_LABEL_MAP[param_name]) def CollectGAMetric(self, category, action, label=VERSION, value=0, execution_time=None, custom_params): """Adds a GA metric with the given parameters to the metrics queue. Args: category: str, the GA Event category. action: str, the GA Event action. label: str, the GA Event label. value: int, the GA Event value. execution_time: int, the execution time to record in ms. custom_params: A dictionary of key, value pairs containing custom metrics and dimensions to send with the GA Event. """ params = [('ec', category), ('ea', action), ('el', label), ('ev', value), (_GA_LABEL_MAP['Timestamp'], _GetTimeInMillis())] params.extend([ (k, v) for k, v in six.iteritems(custom_params) if v is not None ]) params.extend([ (k, v) for k, v in six.iteritems(self.ga_params) if v is not None ]) # Log how long after the start of the program this event happened. if execution_time is None: execution_time = _GetTimeInMillis() - self.start_time params.append((_GA_LABEL_MAP['Execution Time'], execution_time)) data = urllib.parse.urlencode(sorted(params)) self._metrics.append( Metric(endpoint=self.endpoint, method='POST', body=data, user_agent=self.user_agent)) # TODO: Collect CPU usage (Linux-only), latency to first byte, and slowest # thread process. class _PeformanceSummaryParams(object): """This class contains information to create a PerformanceSummary event.""" def __init__(self): self.num_processes = 0 self.num_threads = 0 self.num_retryable_service_errors = 0 self.num_retryable_network_errors = 0 self.provider_types = set() # Store the disk stats at the beginning of the command so we can calculate # time spent on disk I/O. if system_util.IS_LINUX: self.disk_counters_start = system_util.GetDiskCounters() # True if using fan parallelism, when the user specifies the -m option. self.uses_fan = False # True if the command uses slice parallelism. self.uses_slice = False # The total times in seconds spent idle and executing by threads. self.thread_idle_time = 0 self.thread_execution_time = 0 # This maps (process id, thread id) to a _ThreadThroughputInfo object, # keeping track of elapsed time and bytes processed. self.thread_throughputs = defaultdict(self._ThreadThroughputInformation) # Data transfer statistics. self.avg_throughput = None # This is the amount of time spent on the Apply call of cp and rsync. self.total_elapsed_time = None self.total_bytes_transferred = None self.num_objects_transferred = 0 # Information to determine the type of transfer. self.is_daisy_chain = False self.has_file_dst = False self.has_cloud_dst = False self.has_file_src = False self.has_cloud_src = False class _ThreadThroughputInformation(object): """A class to keep track of throughput information for a single thread.""" def __init__(self): self.total_bytes_transferred = 0 self.total_elapsed_time = 0 # Start time in seconds of the current task. None if the thread is not # currently executing a task. self.task_start_time = None # The size of the current task, if any. A FileMessage gives us size at # the beginning of a transfer, but we don't want to count these bytes # until the transfer is complete. self.task_size = None def LogTaskStart(self, start_time, bytes_to_transfer): self.task_start_time = start_time self.task_size = bytes_to_transfer def LogTaskEnd(self, end_time): self.total_elapsed_time += end_time - self.task_start_time self.total_bytes_transferred += self.task_size self.task_start_time = None self.task_size = None def GetThroughput(self): return CalculateThroughput(self.total_bytes_transferred, self.total_elapsed_time) def UpdatePerformanceSummaryParams(self, params): """Updates the _PeformanceSummaryParams object. Args: params: A dictionary of keyword arguments. - uses_fan: True if the command uses fan parallelism. - uses_slice: True if the command uses slice parallelism. - avg_throughput: The average throughput of the data transfer. - is_daisy_chain: True if the transfer uses the daisy-chain method. - has_file_dst: True if the transfer's destination is a file URL. - has_cloud_dst: True if the transfer's destination is in the cloud. - has_file_src: True if the transfer has a file URL as a source. - has_cloud_src: True if the transfer has a cloud URL as a source. - total_elapsed_time: The total amount of time spent on Apply. - total_bytes_transferred: The total number of bytes transferred. - thread_idle_time: The additional amount of time that threads spent idle in Apply. - thread_execution_time: The additional amount of time that threads spent executing in Apply. - num_retryable_service_errors: The additional number of retryable service errors that occurred. - num_retryable_network_errors: The additional number of retryable network errors that occurred. - num_processes: The number of processes used in a call to Apply. - num_threads: The number of threads used in a call to Apply. - num_objects_transferred: The total number of objects transferred, as specified by a ProducerThreadMessage. - provider_types: A list of additional provider types used. - file_message: A FileMessage used to calculate thread throughput and number of objects transferred in the non-parallel case. """ if self.GetGAParam('Command Name') not in ('cp', 'rsync'): return if self.perf_sum_params is None: self.perf_sum_params = self._PeformanceSummaryParams() # Most of the parameters are logged only one time, but FileMessages are much # more common, with a minimum of 2 per file copied. Since FileMessages are # logged on their own, we can avoid performing the other checks. if 'file_message' in params: self._ProcessFileMessage(file_message=params['file_message']) return for param_name, param in six.iteritems(params): # These parameters start in 0 or False state and can be updated to a # non-zero value or True. if param_name in ('uses_fan', 'uses_slice', 'avg_throughput', 'is_daisy_chain', 'has_file_dst', 'has_cloud_dst', 'has_file_src', 'has_cloud_src', 'total_elapsed_time', 'total_bytes_transferred', 'num_objects_transferred'): cur_value = getattr(self.perf_sum_params, param_name) if not cur_value: setattr(self.perf_sum_params, param_name, param) # These parameters need to be incremented. if param_name in ('thread_idle_time', 'thread_execution_time', 'num_retryable_service_errors', 'num_retryable_network_errors'): cur_value = getattr(self.perf_sum_params, param_name) setattr(self.perf_sum_params, param_name, cur_value + param) # Different calls to Apply might use different numbers of processes and # threads; we want the maximum of these. if param_name in ('num_processes', 'num_threads'): cur_value = getattr(self.perf_sum_params, param_name) if cur_value < param: setattr(self.perf_sum_params, param_name, param) # If we have new provider types, add them to our set. if param_name == 'provider_types': self.perf_sum_params.provider_types.update(param) def _ProcessFileMessage(self, file_message): """Processes FileMessages for thread throughput calculations. Update a thread's throughput based on the FileMessage, which marks the start or end of a file or component transfer. The FileMessage provides the number of bytes transferred as well as start and end time. Args: file_message: The FileMessage to process. """ thread_info = (self.perf_sum_params.thread_throughputs[( file_message.process_id, file_message.thread_id)]) if file_message.finished: # If this operation doesn't use parallelism, we manually update the # number of objects transferred rather than relying on # ProducerThreadMessages. if not (self.perf_sum_params.uses_slice or self.perf_sum_params.uses_fan): self.perf_sum_params.num_objects_transferred += 1 thread_info.LogTaskEnd(file_message.time) else: thread_info.LogTaskStart(file_message.time, file_message.size) def _CollectCommandAndErrorMetrics(self): """Aggregates command and error info and adds them to the metrics list.""" # Collect the command metric, including the number of retryable errors. command_name = self.GetGAParam('Command Name') if command_name: self.CollectGAMetric(category=_GA_COMMANDS_CATEGORY, action=command_name, { _GA_LABEL_MAP['Retryable Errors']: sum(self.retryable_errors.values()) }) # Collect the retryable errors. for error_type, num_errors in six.iteritems(self.retryable_errors): self.CollectGAMetric(category=_GA_ERRORRETRY_CATEGORY, action=error_type, {_GA_LABEL_MAP['Retryable Errors']: num_errors}) # Collect the fatal error, if any. fatal_error_type = self.GetGAParam('Fatal Error') if fatal_error_type: self.CollectGAMetric(category=_GA_ERRORFATAL_CATEGORY, action=fatal_error_type) def _CollectPerformanceSummaryMetric(self): """Aggregates PerformanceSummary info and adds the metric to the list.""" if self.perf_sum_params is None: return custom_params = {} # These parameters need no further processing. for attr_name, label in ( ('num_processes', 'Num Processes'), ('num_threads', 'Num Threads'), ('num_retryable_service_errors', 'Num Retryable Service Errors'), ('num_retryable_network_errors', 'Num Retryable Network Errors'), ('avg_throughput', 'Average Overall Throughput'), ('num_objects_transferred', 'Number of Files/Objects Transferred'), ('total_bytes_transferred', 'Size of Files/Objects Transferred'), ): custom_params[_GA_LABEL_MAP[label]] = getattr(self.perf_sum_params, attr_name) # Calculate the disk stats again to calculate deltas of time spent on I/O. if system_util.IS_LINUX: disk_start = self.perf_sum_params.disk_counters_start disk_end = system_util.GetDiskCounters() # Read and write time are the 5th and 6th elements of the stat tuple. custom_params[_GA_LABEL_MAP['Disk I/O Time']] = ( sum([stat[4] + stat[5] for stat in disk_end.values()]) - sum([stat[4] + stat[5] for stat in disk_start.values()])) # Determine source URL type(s). if self.perf_sum_params.has_cloud_src: src_url_type = 'both' if self.perf_sum_params.has_file_src else 'cloud' else: src_url_type = 'file' custom_params[_GA_LABEL_MAP['Source URL Type']] = src_url_type # Determine the type of parallelism used, if any. if self.perf_sum_params.uses_fan: strategy = 'both' if self.perf_sum_params.uses_slice else 'fan' else: strategy = 'slice' if self.perf_sum_params.uses_slice else 'none' custom_params[_GA_LABEL_MAP['Parallelism Strategy']] = strategy # Determine the percentage of time that threads spent idle. total_time = (self.perf_sum_params.thread_idle_time + self.perf_sum_params.thread_execution_time) if total_time: custom_params[_GA_LABEL_MAP['Thread Idle Time Percent']] = ( float(self.perf_sum_params.thread_idle_time) / float(total_time)) # Determine the slowest and fastest thread throughputs. if self.perf_sum_params.thread_throughputs: throughputs = [ thread.GetThroughput() for thread in self.perf_sum_params.thread_throughputs.values() ] custom_params[_GA_LABEL_MAP['Slowest Thread Throughput']] = min( throughputs) custom_params[_GA_LABEL_MAP['Fastest Thread Throughput']] = max( throughputs) # Determine the provider(s) used. custom_params[_GA_LABEL_MAP['Provider Types']] = ','.join( sorted(self.perf_sum_params.provider_types)) # Determine the transfer types. # This maps a transfer type to whether the condition has been met for it. transfer_types = { 'CloudToCloud': self.perf_sum_params.has_cloud_src and self.perf_sum_params.has_cloud_dst, 'CloudToFile': self.perf_sum_params.has_cloud_src and self.perf_sum_params.has_file_dst, 'DaisyChain': self.perf_sum_params.is_daisy_chain, 'FileToCloud': self.perf_sum_params.has_file_src and self.perf_sum_params.has_cloud_dst, 'FileToFile': self.perf_sum_params.has_file_src and self.perf_sum_params.has_file_dst, } action = ','.join( sorted([ transfer_type for transfer_type, cond in six.iteritems(transfer_types) if cond ])) # Use the time spent on Apply rather than the total command execution time # for the execution time metric. This aligns more closely with throughput # and bytes transferred, and the corresponding Command event already tells # us the total time. If PerformanceSummary events are expanded, this may not # reflect one Apply call as commands like rm may call Apply twice. Currently # Apply is timed directly in the RunCommand methods of cp and rsync. apply_execution_time = _GetTimeInMillis( self.perf_sum_params.total_elapsed_time) self.CollectGAMetric(category=_GA_PERFSUM_CATEGORY, action=action, execution_time=apply_execution_time, custom_params) def ReportMetrics(self, wait_for_report=False, log_level=None, log_file_path=None): """Reports the collected metrics using a separate async process. Args: wait_for_report: bool, True if the main process should wait for the subprocess to exit for testing purposes. log_level: int, The subprocess logger's level of debugging for testing purposes. log_file_path: str, The file that the metrics_reporter module should write its logs to. If not supplied, the metrics_reporter module will use a predetermined default path. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. """ self._CollectCommandAndErrorMetrics() self._CollectPerformanceSummaryMetric() if not self._metrics: return if not log_level: log_level = self.logger.getEffectiveLevel() # If this a testing subprocess, we don't want to write to the log file. if os.environ.get('GSUTIL_TEST_ANALYTICS') == '2': log_level = logging.WARN temp_metrics_file = tempfile.NamedTemporaryFile(delete=False) temp_metrics_file_name = six.ensure_str(temp_metrics_file.name) with temp_metrics_file: pickle.dump(self._metrics, temp_metrics_file) logging.debug(self._metrics) self._metrics = [] if log_file_path is not None: # If the path is not None, we'll need to surround the path with quotes # so that the path is passed as a string to the metrics_reporter module. log_file_path = six.ensure_str('r"%s"' % log_file_path) reporting_code = six.ensure_str( 'from gslib.metrics_reporter import ReportMetrics; ' 'ReportMetrics(r"{0}", {1}, log_file_path={2})'.format( temp_metrics_file_name, log_level, log_file_path)) execution_args = [sys.executable, '-c', reporting_code] exec_env = os.environ.copy() exec_env['PYTHONPATH'] = os.pathsep.join(sys.path) # Ensuring submodule (sm) environment keys and values are all str. sm_env = dict() for k, v in six.iteritems(exec_env): sm_env[six.ensure_str(k)] = six.ensure_str(v) try: # In order for Popen to work correctly with Windows/Py3 shell needs # to be True. p = subprocess.Popen(execution_args, env=sm_env, shell=(six.PY3 and system_util.IS_WINDOWS)) self.logger.debug('Metrics reporting process started...') if wait_for_report: # NOTE: p.wait() can cause a deadlock. p.communicate() is recommended. # See python docs for more information. p.communicate() self.logger.debug('Metrics reporting process finished.') except OSError: # This can happen specifically if the Python executable moves between the # start of this process and now. self.logger.debug('Metrics reporting process failed to start.') # Delete the tempfile that would normally be cleaned up in the subprocess. try: os.unlink(temp_metrics_file.name) except: # pylint: disable=bare-except pass def CaptureAndLogException(func): """Function decorator to capture and log any exceptions. This is extra insurance that analytics collection will not hinder the command being run upon an error. Args: func: The function to wrap. Returns: The wrapped function. """ @wraps(func) def Wrapper(args, *kwds): try: return func(args, *kwds) except Exception as e: # pylint:disable=broad-except logger = logging.getLogger('metrics') logger.debug('Exception captured in %s during metrics collection: %s', func.__name__, e) return Wrapper def CaptureThreadStatException(func): """Function decorator to ignore an exception on collecting thread stats. An exception can happen if the thread_stats dictionary's manager gets shutdown before the thread's process is successfully killed. See _ThreadStat.AggregateStat for how we handle that case. Args: func: The function to wrap. Returns: The wrapped function. """ @wraps(func) def Wrapper(args, *kwds): try: return func(args, kwds) except: # pylint:disable=bare-except # Don't surface the exception to the user. pass return Wrapper # Ordering of these wrappers is important, since both have exception handling. # CaptureAndLogException will swallow the exception. @atexit.register @CaptureAndLogException def Shutdown(): """Reports the metrics that were collected upon termination.""" collector = MetricsCollector.GetCollector() if collector: collector.ReportMetrics() @CaptureAndLogException def LogCommandParams(command_name=None, subcommands=None, global_opts=None, sub_opts=None, command_alias=None): """Logs info about the gsutil command being run. This only updates the collector's ga_params. The actual command metric will be collected once ReportMetrics() is called at shutdown. Args: command_name: str, The official command name (e.g. version instead of ver). subcommands: A list of subcommands as strings already validated by RunCommand. We do not log subcommands for the help or test commands. global_opts: A list of string tuples already parsed by __main__. sub_opts: A list of command-level options as string tuples already parsed by RunCommand. command_alias: str, The supported alias that the user inputed. """ collector = MetricsCollector.GetCollector() if not collector: return # Never re-log any parameter that's already there, as we should only be # collecting the global (user-run) command. if command_name and not collector.GetGAParam('Command Name'): collector.ExtendGAParams({_GA_LABEL_MAP['Command Name']: command_name}) if global_opts and not collector.GetGAParam('Global Options'): global_opts_string = ','.join( sorted([opt[0].strip('-') for opt in global_opts])) collector.ExtendGAParams( {_GA_LABEL_MAP['Global Options']: global_opts_string}) # Only log subcommands, suboptions, and command alias if a command has been # logged. command_name = collector.GetGAParam('Command Name') if not command_name: return if subcommands: full_command_name = '{0} {1}'.format(command_name, ' '.join(subcommands)) collector.ExtendGAParams({_GA_LABEL_MAP['Command Name']: full_command_name}) if sub_opts and not collector.GetGAParam('Command-Level Options'): sub_opts_string = ','.join(sorted([opt[0].strip('-') for opt in sub_opts])) collector.ExtendGAParams( {_GA_LABEL_MAP['Command-Level Options']: sub_opts_string}) if command_alias and not collector.GetGAParam('Command Alias'): collector.ExtendGAParams({_GA_LABEL_MAP['Command Alias']: command_alias}) @CaptureAndLogException def LogRetryableError(message): """Logs that a retryable error was caught for a gsutil command. Args: message: The RetryableErrorMessage posted to the global status queue. """ collector = MetricsCollector.GetCollector() if collector: # Update the retryable_errors defaultdict. collector.retryable_errors[message.error_type] += 1 # Update the service error or network error count. if message.is_service_error: LogPerformanceSummaryParams(num_retryable_service_errors=1) else: LogPerformanceSummaryParams(num_retryable_network_errors=1) @CaptureAndLogException def LogFatalError(exception): """Logs that a fatal error was caught for a gsutil command. Args: exception: The exception that the command failed on. """ collector = MetricsCollector.GetCollector() if collector: collector.ExtendGAParams( {_GA_LABEL_MAP['Fatal Error']: exception.__class__.__name__}) @CaptureAndLogException def LogPerformanceSummaryParams(params_to_update): """Logs parameters necessary for a PerformanceSummary. gsutil periodically monitors its own threads; at the end of the execution of each cp/rsync command, it will present a performance summary of the command run. Args: *params_to_update: A dictionary. See UpdatePerformanceSummaryParams for details. The argument ambiguity at this level allows for flexibility in dealing with arguments that are processed similarly. """ collector = MetricsCollector.GetCollector() if collector: collector.UpdatePerformanceSummaryParams(params_to_update) @CaptureAndLogException def CheckAndMaybePromptForAnalyticsEnabling(): """Asks a user to opt-in to data collection if a UUID file does not exist. If the user agrees, generates a UUID file. Will not prompt if part of SDK. """ disable_prompt = boto.config.get_value('GSUtil', 'disable_analytics_prompt') if (not os.path.exists(_UUID_FILE_PATH) and not disable_prompt and not system_util.InvokedViaCloudSdk()): enable_analytics = input('\n' + textwrap.fill( 'gsutil developers rely on user feedback to make improvements to the ' 'tool. Would you like to send anonymous usage statistics to help ' 'improve gsutil? [y/N]') + ' ') text_to_write = _DISABLED_TEXT if enable_analytics.lower()[0] == 'y': text_to_write = uuid.uuid4().hex system_util.CreateDirIfNeeded(os.path.dirname(_UUID_FILE_PATH)) with open(_UUID_FILE_PATH, 'w') as f: f.write(text_to_write) def _GetTimeInMillis(time_in_sec=None): if time_in_sec is None: time_in_sec = time.time() return int(time_in_sec 1000)	38.98 KB	2025-06-25 08:19
📄 metrics_reporter.py # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Script for reporting metrics.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import pickle import random import string import sys _LOG_FILE_BASENAME = 'metrics.log' # pylint:disable=g-import-not-at-top try: from gslib.utils.boto_util import ConfigureCertsFile from gslib.utils.boto_util import GetGsutilStateDir from gslib.utils.boto_util import GetNewHttp from gslib.utils.system_util import CreateDirIfNeeded _LOG_FILE_PARENT_DIR = GetGsutilStateDir() except: # pylint: disable=bare-except # Some environments import their own version of standard Python libraries # which might cause the import of some of our gslib.utils modules to fail # (e.g. if httplib2 is missing, etc). Try these alternative definitions in # such cases. _LOG_FILE_PARENT_DIR = os.path.expanduser(os.path.join('~', '.gsutil')) # Some environments don't have the httplib2 functionality we need. try: import errno # Fall back to httplib (no proxy) if we can't import libraries normally. from six.moves import http_client def GetNewHttp(): """Returns an httplib-based metrics reporter.""" class HttplibReporter(object): def __init__(self): pass # pylint: disable=invalid-name def request(self, endpoint, method=None, body=None, headers=None): # Strip 'https://' https_con = http_client.HTTPSConnection(endpoint[8:].split('/')[0]) https_con.request(method, endpoint, body=body, headers=headers) response = https_con.getresponse() # Return status like an httplib2 response. return ({'status': response.status},) # pylint: enable=invalid-name return HttplibReporter() def ConfigureCertsFile(): pass def CreateDirIfNeeded(dir_path, mode=0o777): """See the same-named method in gslib.utils.system_util.""" if not os.path.exists(dir_path): try: os.makedirs(dir_path, mode) except OSError as e: if e.errno != errno.EEXIST: raise except: # pylint: disable=bare-except sys.exit(0) LOG_FILE_PATH = os.path.join(_LOG_FILE_PARENT_DIR, _LOG_FILE_BASENAME) def ReportMetrics(metrics_file_path, log_level, log_file_path=None): """Sends the specified anonymous usage event to the given analytics endpoint. Args: metrics_file_path: str, File with pickled metrics (list of tuples). log_level: int, The logging level of gsutil's root logger. log_file_path: str, The file that this module should write its logs to. This parameter is intended for use by tests that need to evaluate the contents of the file at this path. """ logger = logging.getLogger() if log_file_path is not None: # Use a separate logger so that we don't add another handler to the default # module-level logger. This is intended to prevent multiple calls from tests # running in parallel from writing output to the same file. new_name = '%s.%s' % (logger.name, ''.join( random.choice(string.ascii_lowercase) for _ in range(8))) logger = logging.getLogger(new_name) log_file_path = log_file_path or LOG_FILE_PATH log_file_parent_dir = os.path.dirname(log_file_path) CreateDirIfNeeded(log_file_parent_dir) handler = logging.FileHandler(log_file_path, mode='w') logger.addHandler(handler) logger.setLevel(log_level) with open(metrics_file_path, 'rb') as metrics_file: metrics = pickle.load(metrics_file) os.remove(metrics_file_path) ConfigureCertsFile() http = GetNewHttp() for metric in metrics: try: headers = {'User-Agent': metric.user_agent} response = http.request(metric.endpoint, method=metric.method, body=metric.body, headers=headers) logger.debug(metric) logger.debug('RESPONSE: %s', response[0]['status']) except Exception as e: # pylint: disable=broad-except logger.debug(e)	4.61 KB	2025-06-25 08:19
📄 metrics_tuple.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Defines a Metric object used for sending information to Google Analytics.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple # A Metric contains all of the information required to post a GA event. It is # not a custom metric, which is a GA term for a specific number to track on the # Analytics dashboard. This is not nested within MetricsCollector to allow # pickle to dump a list of Metrics. Metric = namedtuple( 'Metric', [ # The URL of the request endpoint. 'endpoint', # The HTTP method of request. 'method', # The URL-encoded body to send with the request. 'body', # The user-agent string to send as a header. 'user_agent', ])	1.44 KB	2025-06-25 08:19
📄 name_expansion.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Name expansion iterator and result classes. Name expansion support for the various ways gsutil lets users refer to collections of data (via explicit wildcarding as well as directory, bucket, and bucket subdir implicit wildcarding). This class encapsulates the various rules for determining how these expansions are done. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import logging import os import sys import six from apitools.base.py import encoding import gslib from gslib.exception import CommandException from gslib.exception import NO_URLS_MATCHED_GENERIC from gslib.exception import NO_URLS_MATCHED_TARGET from gslib.plurality_checkable_iterator import PluralityCheckableIterator from gslib.seek_ahead_thread import SeekAheadResult from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages import gslib.wildcard_iterator from gslib.wildcard_iterator import StorageUrlFromString class NameExpansionResult(object): """Holds one fully expanded result from iterating over NameExpansionIterator. The member data in this class need to be pickleable because NameExpansionResult instances are passed through Multiprocessing.Queue. In particular, don't include any boto state like StorageUri, since that pulls in a big tree of objects, some of which aren't pickleable (and even if they were, pickling/unpickling such a large object tree would result in significant overhead). The state held in this object is needed for handling the various naming cases (e.g., copying from a single source URL to a directory generates different dest URL names than copying multiple URLs to a directory, to be consistent with naming rules used by the Unix cp command). For more details see comments in _NameExpansionIterator. """ def __init__(self, source_storage_url, is_multi_source_request, is_multi_top_level_source_request, names_container, expanded_storage_url, expanded_result): """Instantiates a result from name expansion. Args: source_storage_url: StorageUrl that was being expanded. is_multi_source_request: bool indicator whether multiple input URLs or src_url_str expanded to more than one BucketListingRef. is_multi_top_level_source_request: same as is_multi_source_request but measured before recursion. names_container: Bool indicator whether src_url names a container. expanded_storage_url: StorageUrl that was expanded. expanded_result: cloud object metadata in MessageToJson form (for pickleability), if any was iterated; None otherwise. Consumers must call JsonToMessage to get an apitools Object. """ self.source_storage_url = source_storage_url self.is_multi_source_request = is_multi_source_request self.is_multi_top_level_source_request = is_multi_top_level_source_request self.names_container = names_container self.expanded_storage_url = expanded_storage_url self.expanded_result = encoding.MessageToJson( expanded_result) if expanded_result else None def __repr__(self): return '%s' % self.expanded_storage_url class _NameExpansionIterator(object): """Class that iterates over all source URLs passed to the iterator. See details in __iter__ function doc. """ def __init__(self, command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, continue_on_error=False, bucket_listing_fields=None): """Creates a NameExpansionIterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: PluralityCheckableIterator of URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for bucket retrieval. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Examples of _NameExpansionIterator with recursion_requested=True: - Calling with one of the url_strs being 'gs://bucket' will enumerate all top-level objects, as will 'gs://bucket/' and 'gs://bucket/'. - 'gs://bucket/' will enumerate all objects in the bucket. - 'gs://bucket/abc' will enumerate either the single object abc or, if abc is a subdirectory, all objects under abc and any of its subdirectories. - 'gs://bucket/abc/' will enumerate all objects under abc or any of its subdirectories. - 'file:///tmp' will enumerate all files under /tmp, as will 'file:///tmp/' - 'file:///tmp/*' will enumerate all files under /tmp or any of its subdirectories. Example if recursion_requested=False: calling with gs://bucket/abc/ lists matching objects or subdirs, but not sub-subdirs or objects beneath subdirs. Note: In step-by-step comments below we give examples assuming there's a gs://bucket with object paths: abcd/o1.txt abcd/o2.txt xyz/o1.txt xyz/o2.txt and a directory file://dir with file paths: dir/a.txt dir/b.txt dir/c/ """ self.command_name = command_name self.debug = debug self.logger = logger self.gsutil_api = gsutil_api self.url_strs = url_strs self.recursion_requested = recursion_requested self.all_versions = all_versions # Check self.url_strs.HasPlurality() at start because its value can change # if url_strs is itself an iterator. self.url_strs.has_plurality = self.url_strs.HasPlurality() self.cmd_supports_recursion = cmd_supports_recursion self.project_id = project_id self.ignore_symlinks = ignore_symlinks self.continue_on_error = continue_on_error self.bucket_listing_fields = (set(['name']) if not bucket_listing_fields else bucket_listing_fields) # Map holding wildcard strings to use for flat vs subdir-by-subdir listings. # (A flat listing means show all objects expanded all the way down.) self._flatness_wildcard = {True: '*', False: ''} def __iter__(self): """Iterates over all source URLs passed to the iterator. For each src url, expands wildcards, object-less bucket names, subdir bucket names, and directory names, and generates a flat listing of all the matching objects/files. You should instantiate this object using the static factory function NameExpansionIterator, because consumers of this iterator need the PluralityCheckableIterator wrapper built by that function. Yields: gslib.name_expansion.NameExpansionResult. Raises: CommandException: if errors encountered. """ for url_str in self.url_strs: storage_url = StorageUrlFromString(url_str) if (storage_url.IsFileUrl() and (storage_url.IsStream() or storage_url.IsFifo())): if self.url_strs.has_plurality: raise CommandException('Multiple URL strings are not supported ' 'with streaming ("-") URLs or named pipes.') yield NameExpansionResult(source_storage_url=storage_url, is_multi_source_request=False, is_multi_top_level_source_request=False, names_container=False, expanded_storage_url=storage_url, expanded_result=None) continue # Step 1: Expand any explicitly specified wildcards. The output from this # step is an iterator of BucketListingRef. # Starting with gs://buck/abc this step would expand to gs://bucket/abcd src_names_bucket = False if (storage_url.IsCloudUrl() and storage_url.IsBucket() and not self.recursion_requested): # UNIX commands like rm and cp will omit directory references. # If url_str refers only to buckets and we are not recursing, # then produce references of type BUCKET, because they are guaranteed # to pass through Step 2 and be omitted in Step 3. post_step1_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterBuckets(bucket_fields=['id'])) else: # Get a list of objects and prefixes, expanding the top level for # any listed buckets. If our source is a bucket, however, we need # to treat all of the top level expansions as names_container=True. post_step1_iter = PluralityCheckableIterator( self.WildcardIterator(url_str).IterAll( bucket_listing_fields=self.bucket_listing_fields, expand_top_level_buckets=True)) if storage_url.IsCloudUrl() and storage_url.IsBucket(): src_names_bucket = True src_url_expands_to_multi = post_step1_iter.HasPlurality() is_multi_top_level_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # Step 2: Expand bucket subdirs. The output from this # step is an iterator of (names_container, BucketListingRef). # Starting with gs://bucket/abcd this step would expand to: # iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]). subdir_exp_wildcard = self._flatness_wildcard[self.recursion_requested] if self.recursion_requested: post_step2_iter = _ImplicitBucketSubdirIterator( self, post_step1_iter, subdir_exp_wildcard, self.bucket_listing_fields) else: post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter) post_step2_iter = PluralityCheckableIterator(post_step2_iter) # Because we actually perform and check object listings here, this will # raise if url_args includes a non-existent object. However, # plurality_checkable_iterator will buffer the exception for us, not # raising it until the iterator is actually asked to yield the first # result. if post_step2_iter.IsEmpty(): if self.continue_on_error: try: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) except CommandException as e: # Yield a specialized tuple of (exception, stack_trace) to # the wrapping PluralityCheckableIterator. yield (e, sys.exc_info()[2]) else: raise CommandException(NO_URLS_MATCHED_TARGET % url_str) # Step 3. Omit any directories, buckets, or bucket subdirectories for # non-recursive expansions. post_step3_iter = PluralityCheckableIterator( _OmitNonRecursiveIterator(post_step2_iter, self.recursion_requested, self.command_name, self.cmd_supports_recursion, self.logger)) src_url_expands_to_multi = post_step3_iter.HasPlurality() is_multi_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # Step 4. Expand directories and buckets. This step yields the iterated # values. Starting with gs://bucket this step would expand to: # [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt] # Starting with file://dir this step would expand to: # [dir/a.txt, dir/b.txt, dir/c/] for (names_container, blr) in post_step3_iter: src_names_container = src_names_bucket or names_container if blr.IsObject(): yield NameExpansionResult( source_storage_url=storage_url, is_multi_source_request=is_multi_source_request, is_multi_top_level_source_request= is_multi_top_level_source_request, names_container=src_names_container, expanded_storage_url=blr.storage_url, expanded_result=blr.root_object) else: # Use implicit wildcarding to do the enumeration. # At this point we are guaranteed that: # - Recursion has been requested because non-object entries are # filtered in step 3 otherwise. # - This is a prefix or bucket subdirectory because only # non-recursive iterations product bucket references. expanded_url = StorageUrlFromString(blr.url_string) if expanded_url.IsFileUrl(): # Convert dir to implicit recursive wildcard. url_to_iterate = '%s%s%s' % (blr, os.sep, subdir_exp_wildcard) else: # Convert subdir to implicit recursive wildcard. url_to_iterate = expanded_url.CreatePrefixUrl( wildcard_suffix=subdir_exp_wildcard) wc_iter = PluralityCheckableIterator( self.WildcardIterator(url_to_iterate).IterObjects( bucket_listing_fields=self.bucket_listing_fields)) src_url_expands_to_multi = (src_url_expands_to_multi or wc_iter.HasPlurality()) is_multi_source_request = (self.url_strs.has_plurality or src_url_expands_to_multi) # This will be a flattened listing of all underlying objects in the # subdir. for blr in wc_iter: yield NameExpansionResult( source_storage_url=storage_url, is_multi_source_request=is_multi_source_request, is_multi_top_level_source_request=( is_multi_top_level_source_request), names_container=True, expanded_storage_url=blr.storage_url, expanded_result=blr.root_object) def WildcardIterator(self, url_string): """Helper to instantiate gslib.WildcardIterator. Args are same as gslib.WildcardIterator interface, but this method fills in most of the values from instance state. Args: url_string: URL string naming wildcard objects to iterate. Returns: Wildcard iterator over URL string. """ return gslib.wildcard_iterator.CreateWildcardIterator( url_string, self.gsutil_api, all_versions=self.all_versions, project_id=self.project_id, ignore_symlinks=self.ignore_symlinks, logger=self.logger) class SeekAheadNameExpansionIterator(object): """Creates and wraps a _NameExpansionIterator and yields SeekAheadResults. Unlike the NameExpansionIterator, which can make API calls upon __init__ to check for plurality, this iterator does no work until the first iteration occurs. """ def __init__(self, command_name, debug, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, file_size_will_change=False): """Initializes a _NameExpansionIterator with the inputs.""" # Count data bytes only will be transferred/rewritten. # Note that the rsync command uses a different iterator, thus it is not # included here. self.count_data_bytes = (command_name in ('cp', 'mv', 'rewrite') and not file_size_will_change) # Only query the file size if we are counting data bytes, as this may # result in stat'ing files, which is more expensive. bucket_listing_fields = ['size'] if self.count_data_bytes else None self.name_expansion_iterator = _NameExpansionIterator( command_name, debug, logging.getLogger('dummy'), gsutil_api, PluralityCheckableIterator(url_strs), recursion_requested, all_versions=all_versions, cmd_supports_recursion=cmd_supports_recursion, project_id=project_id, ignore_symlinks=ignore_symlinks, continue_on_error=True, bucket_listing_fields=bucket_listing_fields) def __iter__(self): for name_expansion_result in self.name_expansion_iterator: if self.count_data_bytes and name_expansion_result.expanded_result: iterated_metadata = encoding.JsonToMessage( apitools_messages.Object, name_expansion_result.expanded_result) iterated_size = iterated_metadata.size or 0 yield SeekAheadResult(data_bytes=iterated_size) else: yield SeekAheadResult() def NameExpansionIterator(command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=False, cmd_supports_recursion=True, project_id=None, ignore_symlinks=False, continue_on_error=False, bucket_listing_fields=None): """Static factory function for instantiating _NameExpansionIterator. This wraps the resulting iterator in a PluralityCheckableIterator and checks that it is non-empty. Also, allows url_strs to be either an array or an iterator. Args: command_name: name of command being run. debug: Debug level to pass to underlying iterators (range 0..3). logger: logging.Logger object. gsutil_api: Cloud storage interface. Settable for testing/mocking. url_strs: Iterable URL strings needing expansion. recursion_requested: True if -r specified on command-line. If so, listings will be flattened so mapped-to results contain objects spanning subdirectories. all_versions: Bool indicating whether to iterate over all object versions. cmd_supports_recursion: Bool indicating whether this command supports a '-r' flag. Useful for printing helpful error messages. project_id: Project id to use for the current command. ignore_symlinks: If True, ignore symlinks during iteration. continue_on_error: If true, yield no-match exceptions encountered during iteration instead of raising them. bucket_listing_fields: Iterable fields to include in expanded results. Ex. ['name', 'acl']. Underyling iterator is responsible for converting these to list-style format ['items/name', 'items/acl']. If this is None, only the object name is included in the result. Raises: CommandException if underlying iterator is empty. Returns: Name expansion iterator instance. For example semantics, see comments in NameExpansionIterator.__init__. """ url_strs = PluralityCheckableIterator(url_strs) name_expansion_iterator = _NameExpansionIterator( command_name, debug, logger, gsutil_api, url_strs, recursion_requested, all_versions=all_versions, cmd_supports_recursion=cmd_supports_recursion, project_id=project_id, ignore_symlinks=ignore_symlinks, continue_on_error=continue_on_error, bucket_listing_fields=bucket_listing_fields) name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator) if name_expansion_iterator.IsEmpty(): raise CommandException(NO_URLS_MATCHED_GENERIC) return name_expansion_iterator class _NonContainerTuplifyIterator(object): """Iterator that produces the tuple (False, blr) for each iterated value. Used for cases where blr_iter iterates over a set of BucketListingRefs known not to name containers. """ def __init__(self, blr_iter): """Instantiates iterator. Args: blr_iter: iterator of BucketListingRef. """ self.blr_iter = blr_iter def __iter__(self): for blr in self.blr_iter: yield (False, blr) class _OmitNonRecursiveIterator(object): """Iterator wrapper for that omits certain values for non-recursive requests. This iterates over tuples of (names_container, BucketListingReference) and omits directories, prefixes, and buckets from non-recurisve requests so that we can properly calculate whether the source URL expands to multiple URLs. For example, if we have a bucket containing two objects: bucket/foo and bucket/foo/bar and we do a non-recursive iteration, only bucket/foo will be yielded. """ def __init__(self, tuple_iter, recursion_requested, command_name, cmd_supports_recursion, logger): """Instanties the iterator. Args: tuple_iter: Iterator over names_container, BucketListingReference from step 2 in the NameExpansionIterator recursion_requested: If false, omit buckets, dirs, and subdirs command_name: Command name for user messages cmd_supports_recursion: Command recursion support for user messages logger: Log object for user messages """ self.tuple_iter = tuple_iter self.recursion_requested = recursion_requested self.command_name = command_name self.cmd_supports_recursion = cmd_supports_recursion self.logger = logger def __iter__(self): for (names_container, blr) in self.tuple_iter: if not self.recursion_requested and not blr.IsObject(): # At this point we either have a bucket or a prefix, # so if recursion is not requested, we're going to omit it. expanded_url = StorageUrlFromString(blr.url_string) if expanded_url.IsFileUrl(): desc = 'directory' else: desc = blr.type_name if self.cmd_supports_recursion: self.logger.info('Omitting %s "%s". (Did you mean to do %s -r?)', desc, blr.url_string, self.command_name) else: self.logger.info('Omitting %s "%s".', desc, blr.url_string) else: yield (names_container, blr) class _ImplicitBucketSubdirIterator(object): """Iterator wrapper that performs implicit bucket subdir expansion. Each iteration yields tuple (names_container, expanded BucketListingRefs) where names_container is true if URL names a directory, bucket, or bucket subdir. For example, iterating over [BucketListingRef("gs://abc")] would expand to: [BucketListingRef("gs://abc/o1"), BucketListingRef("gs://abc/o2")] if those subdir objects exist, and [BucketListingRef("gs://abc") otherwise. """ def __init__(self, name_exp_instance, blr_iter, subdir_exp_wildcard, bucket_listing_fields): """Instantiates the iterator. Args: name_exp_instance: calling instance of NameExpansion class. blr_iter: iterator over BucketListingRef prefixes and objects. subdir_exp_wildcard: wildcard for expanding subdirectories; expected values are ** if the mapped-to results should contain objects spanning subdirectories, or * if only one level should be listed. bucket_listing_fields: Fields requested in enumerated results. """ self.blr_iter = blr_iter self.name_exp_instance = name_exp_instance self.subdir_exp_wildcard = subdir_exp_wildcard self.bucket_listing_fields = bucket_listing_fields def __iter__(self): for blr in self.blr_iter: if blr.IsPrefix(): # This is a bucket subdirectory, list objects according to the wildcard. prefix_url = StorageUrlFromString(blr.url_string).CreatePrefixUrl( wildcard_suffix=self.subdir_exp_wildcard) implicit_subdir_iterator = PluralityCheckableIterator( self.name_exp_instance.WildcardIterator(prefix_url).IterAll( bucket_listing_fields=self.bucket_listing_fields)) if not implicit_subdir_iterator.IsEmpty(): for exp_blr in implicit_subdir_iterator: yield (True, exp_blr) else: # Prefix that contains no objects, for example in the $folder$ case # or an empty filesystem directory. yield (False, blr) elif blr.IsObject(): yield (False, blr) else: raise CommandException( '_ImplicitBucketSubdirIterator got a bucket reference %s' % blr) class CopyObjectInfo(object): """Represents the information needed for copying a single object. """ def __init__(self, name_expansion_result, exp_dst_url, have_existing_dst_container): """Instantiates the object info from name expansion result and destination. Args: name_expansion_result: StorageUrl that was being expanded. exp_dst_url: StorageUrl of the destination. have_existing_dst_container: Whether exp_url names an existing directory, bucket, or bucket subdirectory. """ self.source_storage_url = name_expansion_result.source_storage_url self.is_multi_source_request = name_expansion_result.is_multi_source_request self.is_multi_top_level_source_request = ( name_expansion_result.is_multi_top_level_source_request) self.names_container = name_expansion_result.names_container self.expanded_storage_url = name_expansion_result.expanded_storage_url self.expanded_result = name_expansion_result.expanded_result self.exp_dst_url = exp_dst_url self.have_existing_dst_container = have_existing_dst_container # Describes the destination information resulted from ExpandUrlToSingleBlr. DestinationInfo = collections.namedtuple( 'DestinationInfo', [ # The expanded destination StorageURL. 'exp_dst_url', # Bool indicating whether the expanded destination names an existing # directory, bucket, or bucket subdirectory. 'have_existing_dst_container', ]) # Describes (NameExpansionIterator, DestinationInfo) tuple. NameExpansionIteratorDestinationTuple = collections.namedtuple( 'NameExpansionIteratorDestinationTuple', [ 'name_expansion_iter', 'destination', ]) class CopyObjectsIterator(six.Iterator): """Iterator wrapper for copying objects and keeping track of source URL types. This is used in the cp command for copying from multiple source to multiple destinations. It takes a list of NameExpansionIteratorDestinationTuple. It wraps them and return CopyObjectInfo objects that wraps NameExpansionResult with the destination. It's used also for collecting analytics PerformanceSummary info, because there may be multiple source URLs and we want to know if any of them are file URLs, if any of them are cloud URLs, if any of them require daisy chain operations, and if any use different providers. The source URL type information will be aggregated at the end of _SequentialApply or _ParallelApply. """ def __init__(self, name_expansion_dest_iter, is_daisy_chain): """Instantiates the iterator. Args: name_expansion_dest_iter: NameExpansionIteratorDestinationTuple iterator. is_daisy_chain: The -D option in cp might have already been specified, in which case we do not need to check again for daisy chain operations. """ self.is_daisy_chain = is_daisy_chain self.has_file_src = False self.has_cloud_src = False self.provider_types = [] self.name_expansion_dest_iter = name_expansion_dest_iter name_expansion_dest_tuple = next(self.name_expansion_dest_iter) self.current_expansion_iter = name_expansion_dest_tuple.name_expansion_iter self.current_destination = name_expansion_dest_tuple.destination def __iter__(self): return self def __next__(self): """Keeps track of URL types as the command iterates over arguments.""" try: name_expansion_result = next(self.current_expansion_iter) except StopIteration: name_expansion_dest_tuple = next(self.name_expansion_dest_iter) self.current_expansion_iter = ( name_expansion_dest_tuple.name_expansion_iter) self.current_destination = name_expansion_dest_tuple.destination return self.__next__() elt = CopyObjectInfo(name_expansion_result, self.current_destination.exp_dst_url, self.current_destination.have_existing_dst_container) # Check if we've seen a file source. if not self.has_file_src and elt.source_storage_url.IsFileUrl(): self.has_file_src = True # Check if we've seen a cloud source. if not self.has_cloud_src and elt.source_storage_url.IsCloudUrl(): self.has_cloud_src = True # Check if we've seen a daisy-chain condition. if self.current_destination.exp_dst_url.IsCloudUrl(): dst_url_scheme = self.current_destination.exp_dst_url.scheme else: dst_url_scheme = None if (not self.is_daisy_chain and dst_url_scheme is not None and elt.source_storage_url.IsCloudUrl() and elt.source_storage_url.scheme != dst_url_scheme): self.is_daisy_chain = True # Check if we've seen a new provider type. if elt.source_storage_url.scheme not in self.provider_types: self.provider_types.append(elt.source_storage_url.scheme) return elt	29.97 KB	2025-06-25 08:19
📄 no_op_auth_plugin.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """No-op authorization plugin allowing boto anonymous access. This allows users to use gsutil for accessing publicly readable buckets and objects without first signing up for an account. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from boto.auth_handler import AuthHandler class NoOpAuth(AuthHandler): """No-op authorization plugin class.""" capability = ['hmac-v4-s3', 's3'] def __init__(self, path, config, provider): pass def add_auth(self, http_request): pass	1.19 KB	2025-06-25 08:19
📄 no_op_credentials.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """No-op implementation of credentials for JSON HTTP requests.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals class NoOpCredentials(object): def __init__(self): pass def authorize(self, http_obj): # pylint: disable=invalid-name return http_obj def set_store(self, store): # pylint: disable=invalid-name pass	1.03 KB	2025-06-25 08:19
📄 parallel_tracker_file.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for composite upload tracker file functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import namedtuple import errno import json import random import six import gslib from gslib.exception import CommandException from gslib.tracker_file import (WriteJsonDataToTrackerFile, RaiseUnwritableTrackerFileException) from gslib.utils.constants import UTF8 ObjectFromTracker = namedtuple('ObjectFromTracker', 'object_name generation') class _CompositeUploadTrackerEntry(object): """Enum class for composite upload tracker file JSON keys.""" COMPONENTS_LIST = 'components' COMPONENT_NAME = 'component_name' COMPONENT_GENERATION = 'component_generation' ENC_SHA256 = 'encryption_key_sha256' PREFIX = 'prefix' def ReadParallelUploadTrackerFile(tracker_file_name, logger): """Read the tracker file from the last parallel composite upload attempt. If it exists, the tracker file is of the format described in WriteParallelUploadTrackerFile or a legacy format. If the file doesn't exist or is formatted incorrectly, then the upload will start from the beginning. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the tracker file to read parse. logger: logging.Logger for outputting log messages. Returns: enc_key_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. component_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. """ enc_key_sha256 = None prefix = None existing_components = [] tracker_file = None # If we already have a matching tracker file, get the serialization data # so that we can resume the upload. try: tracker_file = open(tracker_file_name, 'r') tracker_data = tracker_file.read() tracker_json = json.loads(tracker_data) enc_key_sha256 = tracker_json[_CompositeUploadTrackerEntry.ENC_SHA256] prefix = tracker_json[_CompositeUploadTrackerEntry.PREFIX] for component in tracker_json[_CompositeUploadTrackerEntry.COMPONENTS_LIST]: existing_components.append( ObjectFromTracker( component[_CompositeUploadTrackerEntry.COMPONENT_NAME], component[_CompositeUploadTrackerEntry.COMPONENT_GENERATION])) except IOError as e: # Ignore non-existent file (happens first time a upload is attempted on an # object, or when re-starting an upload after a # ResumableUploadStartOverException), but warn user for other errors. if e.errno != errno.ENOENT: logger.warn( 'Couldn\'t read upload tracker file (%s): %s. Restarting ' 'parallel composite upload from scratch.', tracker_file_name, e.strerror) except (KeyError, ValueError) as e: # Legacy format did not support user-supplied encryption. enc_key_sha256 = None (prefix, existing_components) = _ParseLegacyTrackerData(tracker_data) finally: if tracker_file: tracker_file.close() return (enc_key_sha256, prefix, existing_components) def _ParseLegacyTrackerData(tracker_data): """Parses a legacy parallel composite upload tracker file. Args: tracker_data: Legacy tracker file contents. Returns: component_prefix: The prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. """ # Old tracker files used a non-JSON format. # The first line represents the prefix, followed by line pairs of object_name # and generation. Discard the last blank line. old_tracker_data = tracker_data.split('\n')[:-1] prefix = None existing_components = [] if old_tracker_data: prefix = old_tracker_data[0] i = 1 while i < len(old_tracker_data) - 1: (name, generation) = (old_tracker_data[i], old_tracker_data[i + 1]) if not generation: # Cover the '' case. generation = None existing_components.append(ObjectFromTracker(name, generation)) i += 2 return (prefix, existing_components) def ValidateParallelCompositeTrackerData(tracker_file_name, existing_enc_sha256, existing_prefix, existing_components, current_enc_key_sha256, bucket_url, command_obj, logger, delete_func, delete_exc_handler): """Validates that tracker data matches the current encryption key. If the data does not match, makes a best-effort attempt to delete existing temporary component objects encrypted with the old key. Args: tracker_file_name: String file name of tracker file. existing_enc_sha256: Encryption key SHA256 used to encrypt the existing components, or None if an encryption key was not used. existing_prefix: String prefix used in naming the existing components, or None if no prefix was found. existing_components: A list of ObjectFromTracker objects representing the set of files that have already been uploaded. current_enc_key_sha256: Current Encryption key SHA256 that should be used to encrypt objects. bucket_url: Bucket URL in which the components exist. command_obj: Command class for calls to Apply. logger: logging.Logger for outputting log messages. delete_func: command.Apply-callable function for deleting objects. delete_exc_handler: Exception handler for delete_func. Returns: prefix: existing_prefix, or None if the encryption key did not match. existing_components: existing_components, or empty list if the encryption key did not match. """ if six.PY3: if isinstance(existing_enc_sha256, str): existing_enc_sha256 = existing_enc_sha256.encode(UTF8) if isinstance(current_enc_key_sha256, str): current_enc_key_sha256 = current_enc_key_sha256.encode(UTF8) if existing_prefix and existing_enc_sha256 != current_enc_key_sha256: try: logger.warn( 'Upload tracker file (%s) does not match current encryption ' 'key. Deleting old components and restarting upload from ' 'scratch with a new tracker file that uses the current ' 'encryption key.', tracker_file_name) components_to_delete = [] for component in existing_components: url = bucket_url.Clone() url.object_name = component.object_name url.generation = component.generation command_obj.Apply( delete_func, components_to_delete, delete_exc_handler, arg_checker=gslib.command.DummyArgChecker, parallel_operations_override=command_obj.ParallelOverrideReason.SPEED) except: # pylint: disable=bare-except # Regardless of why we can't clean up old components, need to proceed # with the user's original intent to upload the file, so merely warn. component_names = [ component.object_name for component in existing_components ] logger.warn( 'Failed to delete some of the following temporary objects:\n%s\n' '(Continuing on to re-upload components from scratch.)', '\n'.join(component_names)) # Encryption keys have changed, so the old components and prefix # cannot be used. return (None, []) return (existing_prefix, existing_components) def GenerateComponentObjectPrefix(encryption_key_sha256=None): """Generates a random prefix for component objects. Args: encryption_key_sha256: Encryption key SHA256 that will be used to encrypt the components. This is hashed into the prefix to avoid collision during resumption with a different encryption key. Returns: String prefix for use in the composite upload. """ return str( (random.randint(1, (1010) - 1) + hash(encryption_key_sha256)) % 1010) def WriteComponentToParallelUploadTrackerFile(tracker_file_name, tracker_file_lock, component, logger, encryption_key_sha256=None): """Rewrites an existing tracker file with info about the uploaded component. Follows the format described in _CreateParallelUploadTrackerFile. Args: tracker_file_name: Tracker file to append to. tracker_file_lock: Thread and process-safe Lock protecting the tracker file. component: ObjectFromTracker describing the object that was uploaded. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. """ with tracker_file_lock: (existing_enc_key_sha256, prefix, existing_components) = (ReadParallelUploadTrackerFile( tracker_file_name, logger)) if existing_enc_key_sha256 != encryption_key_sha256: raise CommandException( 'gsutil client error: encryption key SHA256 (%s) in tracker file ' 'does not match encryption key SHA256 (%s) of component %s' % (existing_enc_key_sha256, encryption_key_sha256, component.object_name)) newly_completed_components = [component] completed_components = existing_components + newly_completed_components WriteParallelUploadTrackerFile(tracker_file_name, prefix, completed_components, encryption_key_sha256=encryption_key_sha256) def WriteParallelUploadTrackerFile(tracker_file_name, prefix, components, encryption_key_sha256=None): """Writes information about components that were successfully uploaded. The tracker file is serialized JSON of the form: { "encryption_key_sha256": sha256 hash of encryption key (or null), "prefix": Prefix used for the component objects, "components": [ { "component_name": Component object name, "component_generation": Component object generation (or null), }, ... ] } where N is the number of components that have been successfully uploaded. This function is not thread-safe and must be protected by a lock if called within Command.Apply. Args: tracker_file_name: The name of the parallel upload tracker file. prefix: The generated prefix that used for uploading any existing components. components: A list of ObjectFromTracker objects that were uploaded. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. """ if six.PY3: if isinstance(encryption_key_sha256, bytes): encryption_key_sha256 = encryption_key_sha256.decode('ascii') tracker_components = [] for component in components: tracker_components.append({ _CompositeUploadTrackerEntry.COMPONENT_NAME: component.object_name, _CompositeUploadTrackerEntry.COMPONENT_GENERATION: component.generation }) tracker_file_data = { _CompositeUploadTrackerEntry.COMPONENTS_LIST: tracker_components, _CompositeUploadTrackerEntry.ENC_SHA256: encryption_key_sha256, _CompositeUploadTrackerEntry.PREFIX: prefix } WriteJsonDataToTrackerFile(tracker_file_name, tracker_file_data)	12.14 KB	2025-06-25 08:19
📄 plurality_checkable_iterator.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Iterator wrapper for checking wrapped iterator's emptiness or plurality.""" # TODO: Here and elsewhere (wildcard_iterator, name_expansion), do not reference # __iter__ directly because it causes the first element to be instantiated. # Instead, implement __iter__ as a return self and implement the next() function # which returns (not yields) the values. This necessitates that in the case # of the iterator classes, the iterator is used once per class instantiation # so that next() calls do not collide, but this semantic has been long-assumed # by the iterator classes for the use of __iter__ anyway. from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import sys import six class PluralityCheckableIterator(six.Iterator): """Iterator wrapper class. Allows you to check whether the wrapped iterator is empty and whether it has more than 1 element. This iterator accepts three types of values from the iterator it wraps: 1. A yielded element (this is the normal case). 2. A raised exception, which will be buffered and re-raised when it is reached in this iterator. 3. A yielded tuple of (exception, stack trace), which will be buffered and raised with it is reached in this iterator. """ def __init__(self, it): # Need to get the iterator function here so that we don't immediately # instantiate the first element (which could raise an exception). self.orig_iterator = it self.base_iterator = None self.head = [] self.underlying_iter_empty = False def _PopulateHead(self, num_elements=1): """Populates self.head from the underlying iterator. Args: num_elements: Populate until self.head contains this many elements (or until the underlying iterator runs out). Returns: Number of elements at self.head after execution complete. """ while not self.underlying_iter_empty and len(self.head) < num_elements: try: if not self.base_iterator: self.base_iterator = iter(self.orig_iterator) e = next(self.base_iterator) self.underlying_iter_empty = False if isinstance(e, tuple) and isinstance(e[0], Exception): self.head.append(('exception', e[0], e[1])) else: self.head.append(('element', e)) except StopIteration: # Indicates we can no longer call next() on underlying iterator, but # there could still be elements left to iterate in head. self.underlying_iter_empty = True except Exception as e: # pylint: disable=broad-except # Buffer the exception and raise it when the element is accessed. # Also, preserve the original stack trace, as the stack trace from # within plurality_checkable_iterator.next is not very useful. self.head.append(('exception', e, sys.exc_info()[2])) return len(self.head) def __iter__(self): return self def __next__(self): if self._PopulateHead(): item_tuple = self.head.pop(0) if item_tuple[0] == 'element': return item_tuple[1] else: # buffered exception raise six.reraise(item_tuple[1].__class__, item_tuple[1], item_tuple[2]) raise StopIteration() def IsEmpty(self): return not self._PopulateHead() def HasPlurality(self): # Populate 2 elements (if possible) into head so we can check whether # iterator has more than 1 item remaining. return self._PopulateHead(num_elements=2) > 1 def PeekException(self): """Raises an exception if the first iterated element raised.""" if self._PopulateHead() and self.head[0][0] == 'exception': exception_tuple = self.head[0] raise six.reraise(exception_tuple[1].__class__, exception_tuple[1], exception_tuple[2])	4.4 KB	2025-06-25 08:19
📄 progress_callback.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for progress callbacks.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import time from gslib.thread_message import ProgressMessage from gslib.utils import parallelism_framework_util # Default upper and lower bounds for progress callback frequency. _START_BYTES_PER_CALLBACK = 1024 * 256 _MAX_BYTES_PER_CALLBACK = 1024 * 1024 * 100 _TIMEOUT_SECONDS = 1 # Max width of URL to display in progress indicator. Wide enough to allow # 15 chars for x/y display on an 80 char wide terminal. MAX_PROGRESS_INDICATOR_COLUMNS = 65 class ProgressCallbackWithTimeout(object): """Makes progress callbacks at least once every _TIMEOUT_SECONDS. This prevents wrong throughput calculation while not being excessively overwhelming. """ def __init__(self, total_size, callback_func, start_bytes_per_callback=_START_BYTES_PER_CALLBACK, timeout=_TIMEOUT_SECONDS): """Initializes the callback with timeout. Args: total_size: Total bytes to process. If this is None, size is not known at the outset. callback_func: Func of (int: processed_so_far, int: total_bytes) used to make callbacks. start_bytes_per_callback: Lower bound of bytes per callback. timeout: Number maximum of seconds without a callback. """ self._bytes_per_callback = start_bytes_per_callback self._callback_func = callback_func self._total_size = total_size self._last_time = time.time() self._timeout = timeout self._bytes_processed_since_callback = 0 self._callbacks_made = 0 self._total_bytes_processed = 0 def Progress(self, bytes_processed): """Tracks byte processing progress, making a callback if necessary.""" self._bytes_processed_since_callback += bytes_processed cur_time = time.time() if (self._bytes_processed_since_callback > self._bytes_per_callback or (self._total_size is not None and self._total_bytes_processed + self._bytes_processed_since_callback >= self._total_size) or (self._last_time - cur_time) > self._timeout): self._total_bytes_processed += self._bytes_processed_since_callback # TODO: We check if >= total_size and truncate because JSON uploads count # multipart metadata during their send progress. If the size is unknown, # we can't do this and the progress message will make it appear that we # send more than the original stream. if self._total_size is not None: bytes_sent = min(self._total_bytes_processed, self._total_size) else: bytes_sent = self._total_bytes_processed self._callback_func(bytes_sent, self._total_size) self._bytes_processed_since_callback = 0 self._callbacks_made += 1 self._last_time = cur_time class FileProgressCallbackHandler(object): """Tracks progress info for large operations like file copy or hash. Information is sent to the status_queue, which will print it in the UI Thread. """ def __init__(self, status_queue, start_byte=0, override_total_size=None, src_url=None, component_num=None, dst_url=None, operation_name=None): """Initializes the callback handler. Args: status_queue: Queue for posting status messages for UI display. start_byte: The beginning of the file component, if one is being used. override_total_size: The size of the file component, if one is being used. src_url: FileUrl/CloudUrl representing the source file. component_num: Indicates the component number, if any. dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. operation_name: String representing the operation name """ self._status_queue = status_queue self._start_byte = start_byte self._override_total_size = override_total_size self._component_num = component_num self._src_url = src_url self._dst_url = dst_url self._operation_name = operation_name # Ensures final newline is written once even if we get multiple callbacks. self._last_byte_written = False # Function signature is in boto callback format, which cannot be changed. def call( self, # pylint: disable=invalid-name last_byte_processed, total_size): """Gathers information describing the operation progress. Actual message is printed to stderr by UIThread. Args: last_byte_processed: The last byte processed in the file. For file components, this number should be in the range [start_byte:start_byte + override_total_size]. total_size: Total size of the ongoing operation. """ if self._last_byte_written: return if self._override_total_size: total_size = self._override_total_size parallelism_framework_util.PutToQueueWithTimeout( self._status_queue, ProgressMessage(total_size, last_byte_processed - self._start_byte, self._src_url, time.time(), component_num=self._component_num, operation_name=self._operation_name, dst_url=self._dst_url)) if total_size and last_byte_processed - self._start_byte == total_size: self._last_byte_written = True	6.08 KB	2025-06-25 08:19
📄 project_id.py # -- coding: utf-8 -- # Copyright 2011 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper module for Google Cloud Storage project ids.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto from gslib.cloud_api import ProjectIdException GOOG_PROJ_ID_HDR = 'x-goog-project-id' UNIT_TEST_PROJECT_ID = None # Set to a string value when running unit tests. def PopulateProjectId(project_id=None): """Returns the project_id from the boto config file if one is not provided.""" # Provided project_id takes highest precedence. if project_id: return project_id # If present, return the default project id supplied in the boto config file. default_id = boto.config.get_value('GSUtil', 'default_project_id') if default_id: return default_id # Unit tests shouldn't interact with projects, but some functionality # depends on this method to return a project id. If the default project id # value has been set by the unit test setup, return that id. if UNIT_TEST_PROJECT_ID: return UNIT_TEST_PROJECT_ID raise ProjectIdException('MissingProjectId')	1.69 KB	2025-06-25 08:19
📄 pubsub_api.py # -- coding: utf-8 -- # Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """JSON gsutil Cloud API implementation for Google Cloud Storage.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import json import logging import traceback from apitools.base.py import exceptions as apitools_exceptions from boto import config from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import BadRequestException from gslib.cloud_api import NotFoundException from gslib.cloud_api import PreconditionException from gslib.cloud_api import ServiceException from gslib.gcs_json_credentials import SetUpJsonCredentialsAndCache from gslib.no_op_credentials import NoOpCredentials from gslib.third_party.pubsub_apitools import pubsub_v1_client as apitools_client from gslib.third_party.pubsub_apitools import pubsub_v1_messages as apitools_messages from gslib.utils import system_util from gslib.utils.boto_util import GetCertsFile from gslib.utils.boto_util import GetMaxRetryDelay from gslib.utils.boto_util import GetNewHttp from gslib.utils.boto_util import GetNumRetries from gslib.utils.constants import UTF8 TRANSLATABLE_APITOOLS_EXCEPTIONS = (apitools_exceptions.HttpError) if system_util.InvokedViaCloudSdk(): _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gcloud auth login`') else: _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE = ( 'Insufficient OAuth2 scope to perform this operation. ' 'Please re-run `gsutil config`') class PubsubApi(object): """Wraps calls to the Cloud Pub/Sub v1 interface via apitools.""" def __init__(self, logger=None, credentials=None, debug=0): """Performs necessary setup for interacting with Google Cloud Pub/Sub. Args: logger: logging.logger for outputting log messages. credentials: Credentials to be used for interacting with Google Cloud Pub/Sub debug: Debug level for the API implementation (0..3). """ super(PubsubApi, self).__init__() self.logger = logger self.certs_file = GetCertsFile() self.http = GetNewHttp() self.http_base = 'https://' self.host_base = config.get('Credentials', 'gs_pubsub_host', 'pubsub.googleapis.com') gs_pubsub_port = config.get('Credentials', 'gs_pubsub_port', None) self.host_port = (':' + gs_pubsub_port) if gs_pubsub_port else '' self.url_base = (self.http_base + self.host_base + self.host_port) SetUpJsonCredentialsAndCache(self, logger, credentials=credentials) log_request = (debug >= 3) log_response = (debug >= 3) self.api_client = apitools_client.PubsubV1(url=self.url_base, http=self.http, log_request=log_request, log_response=log_response, credentials=self.credentials) self.num_retries = GetNumRetries() self.api_client.num_retries = self.num_retries self.max_retry_wait = GetMaxRetryDelay() self.api_client.max_retry_wait = self.max_retry_wait if isinstance(self.credentials, NoOpCredentials): # This API key is not secret and is used to identify gsutil during # anonymous requests. self.api_client.AddGlobalParam('key', 'AIzaSyDnacJHrKma0048b13sh8cgxNUwulubmJM') def GetTopic(self, topic_name): request = apitools_messages.PubsubProjectsTopicsGetRequest(topic=topic_name) try: return self.api_client.projects_topics.Get(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def CreateTopic(self, topic_name): topic = apitools_messages.Topic(name=topic_name) try: return self.api_client.projects_topics.Create(topic) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def DeleteTopic(self, topic_name): """Only used in system tests.""" request = apitools_messages.PubsubProjectsTopicsDeleteRequest( topic=topic_name) try: return self.api_client.projects_topics.Delete(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def GetTopicIamPolicy(self, topic_name): request = apitools_messages.PubsubProjectsTopicsGetIamPolicyRequest( resource=topic_name) try: return self.api_client.projects_topics.GetIamPolicy(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def SetTopicIamPolicy(self, topic_name, policy): policy_request = apitools_messages.SetIamPolicyRequest(policy=policy) request = apitools_messages.PubsubProjectsTopicsSetIamPolicyRequest( resource=topic_name, setIamPolicyRequest=policy_request) try: return self.api_client.projects_topics.SetIamPolicy(request) except TRANSLATABLE_APITOOLS_EXCEPTIONS as e: self._TranslateExceptionAndRaise(e, topic_name=topic_name) def _TranslateExceptionAndRaise(self, e, topic_name=None): """Translates an HTTP exception and raises the translated or original value. Args: e: Any Exception. topic_name: Optional topic name in request that caused the exception. Raises: Translated CloudApi exception, or the original exception if it was not translatable. """ if self.logger.isEnabledFor(logging.DEBUG): self.logger.debug('TranslateExceptionAndRaise: %s', traceback.format_exc()) translated_exception = self._TranslateApitoolsException( e, topic_name=topic_name) if translated_exception: raise translated_exception else: raise def _GetMessageFromHttpError(self, http_error): if isinstance(http_error, apitools_exceptions.HttpError): if getattr(http_error, 'content', None): try: json_obj = json.loads(http_error.content.decode(UTF8)) if 'error' in json_obj and 'message' in json_obj['error']: return json_obj['error']['message'] except Exception: # pylint: disable=broad-except # If we couldn't decode anything, just leave the message as None. pass def _GetAcceptableScopesFromHttpError(self, http_error): try: www_authenticate = http_error.response['www-authenticate'] # In the event of a scope error, the www-authenticate field of the HTTP # response should contain text of the form # # 'Bearer realm="https://oauth2.googleapis.com/", # error=insufficient_scope, # scope="${space separated list of acceptable scopes}"' # # Here we use a quick string search to find the scope list, just looking # for a substring with the form 'scope="${scopes}"'. scope_idx = www_authenticate.find('scope="') if scope_idx >= 0: scopes = www_authenticate[scope_idx:].split('"')[1] return 'Acceptable scopes: %s' % scopes except Exception: # pylint: disable=broad-except # Return None if we have any trouble parsing out the acceptable scopes. pass def _TranslateApitoolsException(self, e, topic_name=None): """Translates apitools exceptions into their gsutil equivalents. Args: e: Any exception in TRANSLATABLE_APITOOLS_EXCEPTIONS. topic_name: Optional topic name in request that caused the exception. Returns: ServiceException for translatable exceptions, None otherwise. """ if isinstance(e, apitools_exceptions.HttpError): message = self._GetMessageFromHttpError(e) if e.status_code == 400: # It is possible that the Project ID is incorrect. Unfortunately the # JSON API does not give us much information about what part of the # request was bad. return BadRequestException(message or 'Bad Request', status=e.status_code) elif e.status_code == 401: if 'Login Required' in str(e): return AccessDeniedException(message or 'Access denied: login required.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) elif e.status_code == 403: if 'The account for the specified project has been disabled' in str(e): return AccessDeniedException(message or 'Account disabled.', status=e.status_code) elif 'Daily Limit for Unauthenticated Use Exceeded' in str(e): return AccessDeniedException(message or 'Access denied: quota exceeded. ' 'Is your project ID valid?', status=e.status_code) elif 'User Rate Limit Exceeded' in str(e): return AccessDeniedException( 'Rate limit exceeded. Please retry this ' 'request later.', status=e.status_code) elif 'Access Not Configured' in str(e): return AccessDeniedException( 'Access Not Configured. Please go to the Google Cloud Platform ' 'Console (https://cloud.google.com/console#/project) for your ' 'project, select APIs and Auth and enable the ' 'Google Cloud Pub/Sub API.', status=e.status_code) elif 'insufficient_scope' in str(e): # If the service includes insufficient scope error detail in the # response body, this check can be removed. return AccessDeniedException( _INSUFFICIENT_OAUTH2_SCOPE_MESSAGE, status=e.status_code, body=self._GetAcceptableScopesFromHttpError(e)) else: return AccessDeniedException(message or e.message, status=e.status_code) elif e.status_code == 404: return NotFoundException(message, status=e.status_code) elif e.status_code == 409 and topic_name: return ServiceException('The topic %s already exists.' % topic_name, status=e.status_code) elif e.status_code == 412: return PreconditionException(message, status=e.status_code) return ServiceException(message, status=e.status_code)	11.17 KB	2025-06-25 08:19
📄 resumable_streaming_upload.py # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper class for streaming resumable uploads.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import collections import os import six from gslib.exception import CommandException from gslib.utils.boto_util import GetJsonResumableChunkSize from gslib.utils.constants import UTF8 class ResumableStreamingJsonUploadWrapper(object): """Wraps an input stream in a buffer for resumable uploads. This class takes a non-seekable input stream, buffers it, and exposes it as a stream with limited seek capabilities such that it can be used in a resumable JSON API upload. max_buffer_size bytes of buffering is supported. """ def __init__(self, stream, max_buffer_size, test_small_buffer=False): """Initializes the wrapper. Args: stream: Input stream. max_buffer_size: Maximum size of internal buffer; should be >= the chunk size of the resumable upload API to ensure that at least one full chunk write can be replayed in the event of a server error. test_small_buffer: Skip check for buffer size vs. chunk size, for testing. """ self._orig_fp = stream if not test_small_buffer and max_buffer_size < GetJsonResumableChunkSize(): raise CommandException('Resumable streaming upload created with buffer ' 'size %s, JSON resumable upload chunk size %s. ' 'Buffer size must be >= JSON resumable upload ' 'chunk size to ensure that uploads can be ' 'resumed.' % (max_buffer_size, GetJsonResumableChunkSize())) self._max_buffer_size = max_buffer_size self._buffer = collections.deque() self._buffer_start = 0 self._buffer_end = 0 self._position = 0 @property def mode(self): """Returns the mode of the underlying file descriptor, or None.""" return getattr(self._orig_fp, 'mode', None) def read(self, size=-1): # pylint: disable=invalid-name """"Reads from the wrapped stream. Args: size: The amount of bytes to read. If omitted or negative, the entire contents of the stream will be read and returned. Returns: Bytes from the wrapped stream. """ read_all_bytes = size is None or size < 0 if read_all_bytes: bytes_remaining = self._max_buffer_size else: bytes_remaining = size data = b'' buffered_data = [] if self._position < self._buffer_end: # There was a backwards seek, so read from the buffer first. # TODO: Performance test to validate if it is worth re-aligning # the buffers in this case. Also, seeking through the buffer for # each read on a long catch-up is probably not performant, but we'd # need a more complex data structure than a deque to get around this. pos_in_buffer = self._buffer_start buffer_index = 0 # First, find the start position in the buffer. while pos_in_buffer + len(self._buffer[buffer_index]) < self._position: # When this loop exits, buffer_index will refer to a buffer that # has at least some overlap with self._position, and # pos_in_buffer will be >= self._position pos_in_buffer += len(self._buffer[buffer_index]) buffer_index += 1 # Read until we've read enough or we're out of buffer. while pos_in_buffer < self._buffer_end and bytes_remaining > 0: buffer_len = len(self._buffer[buffer_index]) # This describes how far into the current buffer self._position is. offset_from_position = self._position - pos_in_buffer bytes_available_this_buffer = buffer_len - offset_from_position read_size = min(bytes_available_this_buffer, bytes_remaining) buffered_data.append(self._buffer[buffer_index] [offset_from_position:offset_from_position + read_size]) bytes_remaining -= read_size pos_in_buffer += buffer_len buffer_index += 1 self._position += read_size # At this point we're guaranteed that if there are any bytes left to read, # then self._position == self._buffer_end, and we can read from the # wrapped stream if needed. if read_all_bytes: # TODO: The user is requesting reading until the end of an # arbitrary length stream, which is bad we'll need to return data # with no size limits; if the stream is sufficiently long, we could run # out of memory. We could break this down into smaller reads and # buffer it as we go, but we're still left returning the data all at # once to the caller. We could raise, but for now trust the caller to # be sane and have enough memory to hold the remaining stream contents. new_data = self._orig_fp.read(size) data_len = len(new_data) if not buffered_data: data = new_data else: buffered_data.append(new_data) data = b''.join(buffered_data) self._position += data_len elif bytes_remaining: new_data = self._orig_fp.read(bytes_remaining) if not buffered_data: data = new_data else: buffered_data.append(new_data) data = b''.join(buffered_data) data_len = len(new_data) if data_len: self._position += data_len self._buffer.append(new_data) self._buffer_end += data_len oldest_data = None while self._buffer_end - self._buffer_start > self._max_buffer_size: oldest_data = self._buffer.popleft() self._buffer_start += len(oldest_data) if oldest_data: refill_amount = self._max_buffer_size - (self._buffer_end - self._buffer_start) if refill_amount: self._buffer.appendleft(oldest_data[-refill_amount:]) self._buffer_start -= refill_amount else: if six.PY3: if buffered_data: buffered_data = [ bd.encode(UTF8) if isinstance(bd, str) else bd for bd in buffered_data ] data = b''.join(buffered_data) if buffered_data else b'' return data def tell(self): # pylint: disable=invalid-name """Returns the current stream position.""" return self._position def seekable(self): # pylint: disable=invalid-name """Returns true since limited seek support exists.""" return True def seek(self, offset, whence=os.SEEK_SET): # pylint: disable=invalid-name """Seeks on the buffered stream. Args: offset: The offset to seek to; must be within the buffer bounds. whence: Must be os.SEEK_SET. Raises: CommandException if an unsupported seek mode or position is used. """ if whence == os.SEEK_SET: if offset < self._buffer_start or offset > self._buffer_end: raise CommandException('Unable to resume upload because of limited ' 'buffering available for streaming uploads. ' 'Offset %s was requested, but only data from ' '%s to %s is buffered.' % (offset, self._buffer_start, self._buffer_end)) # Move to a position within the buffer. self._position = offset elif whence == os.SEEK_END: if offset > self._max_buffer_size: raise CommandException('Invalid SEEK_END offset %s on streaming ' 'upload. Only %s can be buffered.' % (offset, self._max_buffer_size)) # Read to the end and rely on buffering to handle the offset. while self.read(self._max_buffer_size): pass # Now we're at the end. self._position -= offset else: raise CommandException('Invalid seek mode on streaming upload. ' '(mode %s, offset %s)' % (whence, offset)) def close(self): # pylint: disable=invalid-name return self._orig_fp.close()	8.53 KB	2025-06-25 08:19
📄 seek_ahead_thread.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Threading code for estimating total work of long-running gsutil commands.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import threading import time from gslib import thread_message from gslib.utils import constants from gslib.utils import parallelism_framework_util _PutToQueueWithTimeout = parallelism_framework_util.PutToQueueWithTimeout class SeekAheadResult(object): """Result class for seek_ahead_iterator results. A class is used instead of a namedtuple, making it easier to document and use default keyword arguments. """ def __init__(self, est_num_ops=1, data_bytes=0): """Create a SeekAheadResult. Args: est_num_ops: Number of operations the iterated result represents. Operation is loosely defined as a single API call for a single object. The total number of API calls may not be known at the time of iteration, so this number is approximate. data_bytes: Number of data bytes that will be transferred (uploaded, downloaded, or rewritten) for this iterated result. """ self.est_num_ops = est_num_ops self.data_bytes = data_bytes class SeekAheadThread(threading.Thread): """Thread to estimate total work to be performed by all processes and threads. Because the ProducerThread can only buffer a certain number of tasks on the global task queue, it cannot reliably provide the total count or size of iterated results for operations involving many iterated arguments until it nears the end of iteration. This thread consumes an iterator that should be functionally identical to the ProducerThread, but iterates to the end without adding tasks to the global task queue in an effort to estimate the amount of total work that the call to Apply will perform. It should be used only for large operations, and thus it is created by the main ProducerThread only when the number of iterated arguments exceeds a threshold. This thread may produce an inaccurate estimate if its iterator produces different results than the iterator used by the ProducerThread. This can happen due to eventual listing consistency or due to the source being modified as iteration occurs. This thread estimates operations for top-level objects only; sub-operations (such as a parallel composite upload) should be reported via the iterator as a single object including the total number of bytes affected. """ def __init__(self, seek_ahead_iterator, cancel_event, status_queue): """Initializes the seek ahead thread. Args: seek_ahead_iterator: Iterator matching the ProducerThread's args_iterator, but returning only object name and/or size in the result. cancel_event: threading.Event for signaling the seek-ahead iterator to terminate. status_queue: Status queue for posting summary of fully iterated results. """ super(SeekAheadThread, self).__init__() self.status_queue = status_queue self.seek_ahead_iterator = seek_ahead_iterator self.cancel_event = cancel_event # For unit-testing only; use cancel_event to stop the thread. self.terminate = False self.start() def run(self): num_objects = 0 num_data_bytes = 0 try: for seek_ahead_result in self.seek_ahead_iterator: if self.terminate: return # Periodically check to see if the ProducerThread has actually # completed, at which point providing an estimate is no longer useful. if (num_objects % constants.NUM_OBJECTS_PER_LIST_PAGE) == 0: if self.cancel_event.isSet(): return num_objects += seek_ahead_result.est_num_ops num_data_bytes += seek_ahead_result.data_bytes except OSError as e: # This can happen because the seek_ahead_iterator races with the command # being run (e.g., a gsutil mv command, which iterates over and moves # files while the estimator is concurrently iterating over the files to # count them). If this happens return here without calling # _PutToQueueWithTimeout, so we don't signal the UI thread that seek ahead # work has completed its estimation. This will cause no estimate to be # printed, but the command will continue to execute as usual. return if self.cancel_event.isSet(): return _PutToQueueWithTimeout( self.status_queue, thread_message.SeekAheadMessage(num_objects, num_data_bytes, time.time()))	5.11 KB	2025-06-25 08:19
📄 sig_handling.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Signal handling functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import logging import os import re import signal import sys import traceback from gslib import metrics from gslib.exception import ControlCException from gslib.utils.constants import UTF8 from gslib.utils.system_util import IS_WINDOWS if IS_WINDOWS: import ctypes # pylint: disable=g-import-not-at-top # Maps from signal_num to list of signal handlers to call. _non_final_signal_handlers = {} # Maps from signal_num to the final signal handler (if any) that should be # called for that signal. _final_signal_handlers = {} def RegisterSignalHandler(signal_num, handler, is_final_handler=False): """Registers a handler for signal signal_num. Unlike calling signal.signal(): - This function can be called from any thread (and will cause the handler to be run by the main thread when the signal is received). - Handlers are cumulative: When a given signal is received, all registered handlers will be executed (with the exception that only the last handler to register with is_final_handler=True will be called). Handlers should make no ordering assumptions, other than that the last handler to register with is_final_handler=True will be called after all the other handlers. Args: signal_num: The signal number with which to associate handler. handler: The handler. is_final_handler: Bool indicator whether handler should be called last among all the handlers for this signal_num. The last handler to register this way survives; other handlers registered with is_final_handler=True will not be called when the signal is received. Raises: RuntimeError: if attempt is made to register a signal_num not in GetCaughtSignals. """ if signal_num not in GetCaughtSignals(): raise RuntimeError('Attempt to register handler (%s) for signal %d, which ' 'is not in GetCaughtSignals' % (handler, signal_num)) if is_final_handler: _final_signal_handlers[signal_num] = handler else: _non_final_signal_handlers[signal_num].append(handler) def _SignalHandler(signal_num, cur_stack_frame): """Global signal handler. When a signal is caught we execute each registered handler for that signal. Args: signal_num: Signal that was caught. cur_stack_frame: Unused. """ if signal_num in _non_final_signal_handlers: for handler in _non_final_signal_handlers[signal_num]: handler(signal_num, cur_stack_frame) if signal_num in _final_signal_handlers: _final_signal_handlers[signal_num](signal_num, cur_stack_frame) def InitializeSignalHandling(): """Initializes global signal handling. Sets up global signal handler for each signal we handle. """ for signal_num in GetCaughtSignals(): _non_final_signal_handlers[signal_num] = [] # Make main signal handler catch the signal. signal.signal(signal_num, _SignalHandler) def GetCaughtSignals(): """Returns terminating signals that can be caught on this OS platform.""" signals = [signal.SIGINT, signal.SIGTERM] if not IS_WINDOWS: # Windows doesn't have SIGQUIT. signals.append(signal.SIGQUIT) return signals def KillProcess(pid): """Make best effort to kill the given process. We ignore all exceptions so a caller looping through a list of processes will continue attempting to kill each, even if one encounters a problem. Args: pid: The process ID. """ try: # os.kill doesn't work in Python3 versions before 3.2. if IS_WINDOWS and ((3, 0) <= sys.version_info[:3] < (3, 2)): kernel32 = ctypes.windll.kernel32 handle = kernel32.OpenProcess(1, 0, pid) kernel32.TerminateProcess(handle, 0) else: os.kill(pid, signal.SIGKILL) except: # pylint: disable=bare-except pass # pylint: disable=unused-argument def MultithreadedMainSignalHandler(signal_num, cur_stack_frame): """Final signal handler for multi-threaded main process.""" if signal_num == signal.SIGINT: if logging.getLogger().isEnabledFor(logging.DEBUG): stack_trace = ''.join(traceback.format_list(traceback.extract_stack())) err = ('DEBUG: Caught CTRL-C (signal %d) - Exception stack trace:\n' ' %s' % (signal_num, re.sub('\\n', '\n ', stack_trace))) try: sys.stderr.write(err.encode(UTF8)) except (UnicodeDecodeError, TypeError) as e: # Can happen when outputting invalid Unicode filenames. # or in python3, where stderr only accepts arguments of type # str, not bytes. sys.stderr.write(err) else: sys.stderr.write('Caught CTRL-C (signal %d) - exiting\n' % signal_num) metrics.LogFatalError(exception=ControlCException()) metrics.Shutdown() KillProcess(os.getpid()) def ChildProcessSignalHandler(signal_num, cur_stack_frame): """Final signal handler for child processes.""" KillProcess(os.getpid()) # pylint: enable=unused-argument	5.64 KB	2025-06-25 08:19
📄 storage_uri_builder.py # -- coding: utf-8 -- # Copyright 2012 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Class that holds state for instantiating StorageUri objects. The StorageUri func defined in this class uses that state (bucket_storage_uri_class and debug) needed plus gsutil default flag values to instantiate this frequently constructed object with just one param for most cases. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import boto class StorageUriBuilder(object): """Class for instantiating StorageUri objects.""" def __init__(self, debug, bucket_storage_uri_class): """Initializes the builder. Args: debug: Debug level to pass in to boto connection (range 0..3). bucket_storage_uri_class: Class to instantiate for cloud StorageUris. Settable for testing/mocking. """ self.bucket_storage_uri_class = bucket_storage_uri_class self.debug = debug def StorageUri(self, uri_str): """Instantiates StorageUri using class state and gsutil default flag values. Args: uri_str: StorageUri naming bucket or object. Returns: boto.StorageUri for given uri_str. Raises: InvalidUriError: if uri_str not valid. """ return boto.storage_uri( uri_str, 'file', debug=self.debug, validate=False, bucket_storage_uri_class=self.bucket_storage_uri_class, suppress_consec_slashes=False)	2.02 KB	2025-06-25 08:19
📄 storage_url.py # -- coding: utf-8 -- # Copyright 2013 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """File and Cloud URL representation classes.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import re import stat import sys from gslib.exception import CommandException from gslib.exception import InvalidUrlError from gslib.utils import system_util from gslib.utils import text_util # Matches provider strings of the form 'gs://' PROVIDER_REGEX = re.compile(r'(?P<provider>[^:])://$') # Matches bucket strings of the form 'gs://bucket' BUCKET_REGEX = re.compile(r'(?P<provider>[^:])://(?P<bucket>[^/])/{0,1}$') # Matches object strings of the form 'gs://bucket/obj' OBJECT_REGEX = re.compile( r'(?P<provider>[^:])://(?P<bucket>[^/])/(?P<object>.)') # Matches versioned object strings of the form 'gs://bucket/obj#1234' GS_GENERATION_REGEX = re.compile(r'(?P<object>.+)#(?P<generation>[0-9]+)$') # Matches versioned object strings of the form 's3://bucket/obj#NULL' S3_VERSION_REGEX = re.compile(r'(?P<object>.+)#(?P<version_id>.+)$') # Matches file strings of the form 'file://dir/filename' FILE_OBJECT_REGEX = re.compile(r'([^:]://)(?P<filepath>.)') # Regex to determine if a string contains any wildcards. WILDCARD_REGEX = re.compile(r'[?\[\]]') RELATIVE_PATH_SYMBOLS = frozenset(['.', '..']) class StorageUrl(object): """Abstract base class for file and Cloud Storage URLs.""" def Clone(self): raise NotImplementedError('Clone not overridden') def IsFileUrl(self): raise NotImplementedError('IsFileUrl not overridden') def IsCloudUrl(self): raise NotImplementedError('IsCloudUrl not overridden') def IsStream(): raise NotImplementedError('IsStream not overridden') def IsFifo(self): raise NotImplementedError('IsFifo not overridden') def CreatePrefixUrl(self, wildcard_suffix=None): """Returns a prefix of this URL that can be used for iterating. Args: wildcard_suffix: If supplied, this wildcard suffix will be appended to the prefix with a trailing slash before being returned. Returns: A prefix of this URL that can be used for iterating. If this URL contains a trailing slash, it will be stripped to create the prefix. This helps avoid infinite looping when prefixes are iterated, but preserves other slashes so that objects with '/' in the name are handled properly. For example, when recursively listing a bucket with the following contents: gs://bucket// <-- object named slash gs://bucket//one-dir-deep a top-level expansion with '/' as a delimiter will result in the following URL strings: 'gs://bucket//' : OBJECT 'gs://bucket//' : PREFIX If we right-strip all slashes from the prefix entry and add a wildcard suffix, we will get 'gs://bucket/' which will produce identical results (and infinitely recurse). Example return values: ('gs://bucket/subdir/', '') becomes 'gs://bucket/subdir/' ('gs://bucket/', '') becomes 'gs://bucket/' ('gs://bucket/', None) becomes 'gs://bucket' ('gs://bucket/subdir//', '') becomes 'gs://bucket/subdir//' ('gs://bucket/subdir///', '') becomes 'gs://bucket/subdir///' ('gs://bucket/subdir/', '') where 'subdir/' is an object becomes 'gs://bucket/subdir/', but iterating on this will return 'subdir/' as a BucketListingObject, so we will not recurse on it as a subdir during listing. """ raise NotImplementedError('CreatePrefixUrl not overridden') def _WarnIfUnsupportedDoubleWildcard(self): """Warn if use may lead to undefined results.""" # Accepted 'url_string' values with '', where '^' = start, and '$' = end. # - ^$ # - ^/ # - /$ # - // if not self.object_name: return delimiter_bounded_url = self.delim + self.object_name + self.delim split_url = delimiter_bounded_url.split( '{delim}{delim}'.format(delim=self.delim)) removed_correct_double_wildcards_url_string = ''.join(split_url) if '' in removed_correct_double_wildcards_url_string: # Found a center '' not in the format '//'. # Not using logger.warning b/c it's too much overhead to pass the logger # object to every StorageUrl. sys.stderr.write( '** behavior is undefined if directly preceeded or followed by' ' with characters other than / in the cloud and {} locally.'.format( os.sep)) @property def url_string(self): raise NotImplementedError('url_string not overridden') @property def versionless_url_string(self): raise NotImplementedError('versionless_url_string not overridden') def __eq__(self, other): return isinstance(other, StorageUrl) and self.url_string == other.url_string def __hash__(self): return hash(self.url_string) class _FileUrl(StorageUrl): """File URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) file URL string. Depending on the string contents, this class represents one or more directories or files. For File URLs, scheme is always file, bucket_name is always blank, and object_name contains the file/directory path. """ def __init__(self, url_string, is_stream=False, is_fifo=False): self.scheme = 'file' self.delim = os.sep self.bucket_name = '' # If given a URI that starts with "<scheme>://", the object name should not # include that prefix. match = FILE_OBJECT_REGEX.match(url_string) if match and match.lastindex == 2: self.object_name = match.group(2) else: self.object_name = url_string # On Windows, the pathname component separator is "\" instead of "/". If we # find an occurrence of "/", replace it with "\" so that other logic can # rely on being able to split pathname components on `os.sep`. if system_util.IS_WINDOWS: self.object_name = self.object_name.replace('/', '\\') self.generation = None self.is_stream = is_stream self.is_fifo = is_fifo self._WarnIfUnsupportedDoubleWildcard() def Clone(self): return _FileUrl(self.url_string) def IsFileUrl(self): return True def IsCloudUrl(self): return False def IsStream(self): return self.is_stream def IsFifo(self): return self.is_fifo def IsDirectory(self): return (not self.IsStream() and not self.IsFifo() and os.path.isdir(self.object_name)) def CreatePrefixUrl(self, wildcard_suffix=None): return self.url_string @property def url_string(self): return '%s://%s' % (self.scheme, self.object_name) @property def versionless_url_string(self): return self.url_string def __str__(self): return self.url_string class _CloudUrl(StorageUrl): """Cloud URL class providing parsing and convenience methods. This class assists with usage and manipulation of an (optionally wildcarded) cloud URL string. Depending on the string contents, this class represents a provider, bucket(s), or object(s). This class operates only on strings. No cloud storage API calls are made from this class. """ def __init__(self, url_string): self.scheme = None self.delim = '/' self.bucket_name = None self.object_name = None self.generation = None provider_match = PROVIDER_REGEX.match(url_string) bucket_match = BUCKET_REGEX.match(url_string) if provider_match: self.scheme = provider_match.group('provider') elif bucket_match: self.scheme = bucket_match.group('provider') self.bucket_name = bucket_match.group('bucket') else: object_match = OBJECT_REGEX.match(url_string) if object_match: self.scheme = object_match.group('provider') self.bucket_name = object_match.group('bucket') self.object_name = object_match.group('object') if self.object_name == '.' or self.object_name == '..': raise InvalidUrlError('%s is an invalid root-level object name' % self.object_name) if self.scheme == 'gs': generation_match = GS_GENERATION_REGEX.match(self.object_name) if generation_match: self.object_name = generation_match.group('object') self.generation = generation_match.group('generation') elif self.scheme == 's3': version_match = S3_VERSION_REGEX.match(self.object_name) if version_match: self.object_name = version_match.group('object') self.generation = version_match.group('version_id') else: raise InvalidUrlError( 'CloudUrl: URL string %s did not match URL regex' % url_string) if url_string[(len(self.scheme) + len('://')):].startswith(self.delim): raise InvalidUrlError( 'Cloud URL scheme should be followed by colon and two slashes: "://".' ' Found: "{}"'.format(url_string)) self._WarnIfUnsupportedDoubleWildcard() def Clone(self): return _CloudUrl(self.url_string) def IsFileUrl(self): return False def IsCloudUrl(self): return True def IsStream(self): raise NotImplementedError('IsStream not supported on CloudUrl') def IsFifo(self): raise NotImplementedError('IsFifo not supported on CloudUrl') def IsBucket(self): return bool(self.bucket_name and not self.object_name) def IsObject(self): return bool(self.bucket_name and self.object_name) def HasGeneration(self): return bool(self.generation) def IsProvider(self): return bool(self.scheme and not self.bucket_name) def CreatePrefixUrl(self, wildcard_suffix=None): prefix = StripOneSlash(self.versionless_url_string) if wildcard_suffix: prefix = '%s/%s' % (prefix, wildcard_suffix) return prefix @property def bucket_url_string(self): return '%s://%s/' % (self.scheme, self.bucket_name) @property def url_string(self): url_str = self.versionless_url_string if self.HasGeneration(): url_str += '#%s' % self.generation return url_str @property def versionless_url_string(self): if self.IsProvider(): return '%s://' % self.scheme elif self.IsBucket(): return self.bucket_url_string return '%s://%s/%s' % (self.scheme, self.bucket_name, self.object_name) def __str__(self): return self.url_string def GetSchemeFromUrlString(url_str): """Returns scheme component of a URL string.""" end_scheme_idx = url_str.find('://') if end_scheme_idx == -1: # File is the default scheme. return 'file' else: return url_str[0:end_scheme_idx].lower() def IsKnownUrlScheme(scheme_str): return scheme_str in ('file', 's3', 'gs') def _GetPathFromUrlString(url_str): """Returns path component of a URL string.""" end_scheme_idx = url_str.find('://') if end_scheme_idx == -1: return url_str else: return url_str[end_scheme_idx + 3:] def ContainsWildcard(url_string): """Checks whether url_string contains a wildcard. Args: url_string: URL string to check. Returns: bool indicator. """ return bool(WILDCARD_REGEX.search(url_string)) def GenerationFromUrlAndString(url, generation): """Decodes a generation from a StorageURL and a generation string. This is used to represent gs and s3 versioning. Args: url: StorageUrl representing the object. generation: Long or string representing the object's generation or version. Returns: Valid generation string for use in URLs. """ if url.scheme == 's3' and generation: return text_util.DecodeLongAsString(generation) return generation def HaveFileUrls(args_to_check): """Checks whether args_to_check contain any file URLs. Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any file URLs. """ for url_str in args_to_check: storage_url = StorageUrlFromString(url_str) if storage_url.IsFileUrl(): return True return False def HaveProviderUrls(args_to_check): """Checks whether args_to_check contains any provider URLs (like 'gs://'). Args: args_to_check: Command-line argument subset to check. Returns: True if args_to_check contains any provider URLs. """ for url_str in args_to_check: storage_url = StorageUrlFromString(url_str) if storage_url.IsCloudUrl() and storage_url.IsProvider(): return True return False def IsCloudSubdirPlaceholder(url, blr=None): """Determines if a StorageUrl is a cloud subdir placeholder. This function is needed because GUI tools (like the GCS cloud console) allow users to create empty "folders" by creating a placeholder object; and parts of gsutil need to treat those placeholder objects specially. For example, gsutil rsync needs to avoid downloading those objects because they can cause conflicts (see comments in rsync command for details). We currently detect two cases: - Cloud objects whose name ends with '_$folder$' - Cloud objects whose name ends with '/' Args: url: (gslib.storage_url.StorageUrl) The URL to be checked. blr: (gslib.BucketListingRef or None) The blr to check, or None if not available. If `blr` is None, size won't be checked. Returns: (bool) True if the URL is a cloud subdir placeholder, otherwise False. """ if not url.IsCloudUrl(): return False url_str = url.url_string if url_str.endswith('_$folder$'): return True if blr and blr.IsObject(): size = blr.root_object.size else: size = 0 return size == 0 and url_str.endswith('/') def IsFileUrlString(url_str): """Returns whether a string is a file URL.""" return GetSchemeFromUrlString(url_str) == 'file' def StorageUrlFromString(url_str): """Static factory function for creating a StorageUrl from a string.""" scheme = GetSchemeFromUrlString(url_str) if not IsKnownUrlScheme(scheme): raise InvalidUrlError('Unrecognized scheme "%s"' % scheme) if scheme == 'file': path = _GetPathFromUrlString(url_str) is_stream = (path == '-') is_fifo = False try: is_fifo = stat.S_ISFIFO(os.stat(path).st_mode) except OSError: pass return _FileUrl(url_str, is_stream=is_stream, is_fifo=is_fifo) return _CloudUrl(url_str) def StripOneSlash(url_str): if url_str and url_str.endswith('/'): return url_str[:-1] return url_str def UrlsAreForSingleProvider(url_args): """Tests whether the URLs are all for a single provider. Args: url_args: (Iterable[str]) Collection of strings to check. Returns: True if all URLs are for single provider; False if `url_args` was empty (as this would not result in a single unique provider) or URLs targeted multiple unique providers. """ provider = None url = None for url_str in url_args: url = StorageUrlFromString(url_str) if not provider: provider = url.scheme elif url.scheme != provider: return False return provider is not None def UrlsAreMixOfBucketsAndObjects(urls): """Tests whether the URLs are a mix of buckets and objects. Args: url_args: (Iterable[gslib.storage_url.StorageUrl]) Collection of URLs to check. Returns: True if URLs are a mix of buckets and objects. False if URLs are all buckets or all objects. None if invalid Cloud URLs are included. """ if all(url.IsCloudUrl() for url in urls): are_buckets = list(map(lambda x: x.IsBucket(), urls)) return any(are_buckets) and not all(are_buckets) def RaiseErrorIfUrlsAreMixOfBucketsAndObjects(urls, recursion_requested): """Raises error if mix of buckets and objects adjusted for recursion.""" if UrlsAreMixOfBucketsAndObjects(urls) and not recursion_requested: raise CommandException('Cannot operate on a mix of buckets and objects.')	16.16 KB	2025-06-25 08:19
📄 tab_complete.py # -- coding: utf-8 -- # Copyright 2014 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shell tab completion.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import itertools import json import threading import time import boto from boto.gs.acl import CannedACLStrings from gslib.storage_url import IsFileUrlString from gslib.storage_url import StorageUrlFromString from gslib.storage_url import StripOneSlash from gslib.utils.boto_util import GetTabCompletionCacheFilename from gslib.utils.boto_util import GetTabCompletionLogFilename from gslib.wildcard_iterator import CreateWildcardIterator TAB_COMPLETE_CACHE_TTL = 15 _TAB_COMPLETE_MAX_RESULTS = 1000 _TIMEOUT_WARNING = """ Tab completion aborted (took >%ss), you may complete the command manually. The timeout can be adjusted in the gsutil configuration file. """.rstrip() class CompleterType(object): CLOUD_BUCKET = 'cloud_bucket' CLOUD_OBJECT = 'cloud_object' CLOUD_OR_LOCAL_OBJECT = 'cloud_or_local_object' LOCAL_OBJECT = 'local_object' LOCAL_OBJECT_OR_CANNED_ACL = 'local_object_or_canned_acl' NO_OP = 'no_op' class LocalObjectCompleter(object): """Completer object for local files.""" def __init__(self): # This is only safe to import if argcomplete is present in the install # (which happens for Cloud SDK installs), so import on usage, not on load. # pylint: disable=g-import-not-at-top from argcomplete.completers import FilesCompleter self.files_completer = FilesCompleter() def __call__(self, prefix, kwargs): return self.files_completer(prefix, kwargs) class LocalObjectOrCannedACLCompleter(object): """Completer object for local files and canned ACLs. Currently, only Google Cloud Storage canned ACL names are supported. """ def __init__(self): self.local_object_completer = LocalObjectCompleter() def __call__(self, prefix, kwargs): local_objects = self.local_object_completer(prefix, kwargs) canned_acls = [acl for acl in CannedACLStrings if acl.startswith(prefix)] return local_objects + canned_acls class TabCompletionCache(object): """Cache for tab completion results.""" def __init__(self, prefix, results, timestamp, partial_results): self.prefix = prefix self.results = results self.timestamp = timestamp self.partial_results = partial_results @staticmethod def LoadFromFile(filename): """Instantiates the cache from a file. Args: filename: The file to load. Returns: TabCompletionCache instance with loaded data or an empty cache if the file cannot be loaded """ try: with open(filename, 'r') as fp: cache_dict = json.loads(fp.read()) prefix = cache_dict['prefix'] results = cache_dict['results'] timestamp = cache_dict['timestamp'] partial_results = cache_dict['partial-results'] except Exception: # pylint: disable=broad-except # Guarding against incompatible format changes in the cache file. # Erring on the side of not breaking tab-completion in case of cache # issues. prefix = None results = [] timestamp = 0 partial_results = False return TabCompletionCache(prefix, results, timestamp, partial_results) def GetCachedResults(self, prefix): """Returns the cached results for prefix or None if not in cache.""" current_time = time.time() if current_time - self.timestamp >= TAB_COMPLETE_CACHE_TTL: return None results = None if prefix == self.prefix: results = self.results elif (not self.partial_results and prefix.startswith(self.prefix) and prefix.count('/') == self.prefix.count('/')): results = [x for x in self.results if x.startswith(prefix)] if results is not None: # Update cache timestamp to make sure the cache entry does not expire if # the user is performing multiple completions in a single # bucket/subdirectory since we can answer these requests from the cache. # e.g. gs://prefix<tab> -> gs://prefix-mid<tab> -> gs://prefix-mid-suffix self.timestamp = time.time() return results def UpdateCache(self, prefix, results, partial_results): """Updates the in-memory cache with the results for the given prefix.""" self.prefix = prefix self.results = results self.partial_results = partial_results self.timestamp = time.time() def WriteToFile(self, filename): """Writes out the cache to the given file.""" json_str = json.dumps({ 'prefix': self.prefix, 'results': self.results, 'partial-results': self.partial_results, 'timestamp': self.timestamp, }) try: with open(filename, 'w') as fp: fp.write(json_str) except IOError: pass class CloudListingRequestThread(threading.Thread): """Thread that performs a listing request for the given URL string.""" def __init__(self, wildcard_url_str, gsutil_api): """Instantiates Cloud listing request thread. Args: wildcard_url_str: The URL to list. gsutil_api: gsutil Cloud API instance to use. """ super(CloudListingRequestThread, self).__init__() self.daemon = True self._wildcard_url_str = wildcard_url_str self._gsutil_api = gsutil_api self.results = None def run(self): it = CreateWildcardIterator( self._wildcard_url_str, self._gsutil_api).IterAll(bucket_listing_fields=['name']) self.results = [ str(c) for c in itertools.islice(it, _TAB_COMPLETE_MAX_RESULTS) ] class TimeoutError(Exception): pass class CloudObjectCompleter(object): """Completer object for Cloud URLs.""" def __init__(self, gsutil_api, bucket_only=False): """Instantiates completer for Cloud URLs. Args: gsutil_api: gsutil Cloud API instance to use. bucket_only: Whether the completer should only match buckets. """ self._gsutil_api = gsutil_api self._bucket_only = bucket_only def _PerformCloudListing(self, wildcard_url, timeout): """Perform a remote listing request for the given wildcard URL. Args: wildcard_url: The wildcard URL to list. timeout: Time limit for the request. Returns: Cloud resources matching the given wildcard URL. Raises: TimeoutError: If the listing does not finish within the timeout. """ request_thread = CloudListingRequestThread(wildcard_url, self._gsutil_api) request_thread.start() request_thread.join(timeout) if request_thread.is_alive(): # This is only safe to import if argcomplete is present in the install # (which happens for Cloud SDK installs), so import on usage, not on load. # pylint: disable=g-import-not-at-top import argcomplete argcomplete.warn(_TIMEOUT_WARNING % timeout) raise TimeoutError() results = request_thread.results return results def __call__(self, prefix, *kwargs): if not prefix: prefix = 'gs://' elif IsFileUrlString(prefix): return [] wildcard_url = prefix + '' url = StorageUrlFromString(wildcard_url) if self._bucket_only and not url.IsBucket(): return [] timeout = boto.config.getint('GSUtil', 'tab_completion_timeout', 5) if timeout == 0: return [] start_time = time.time() cache = TabCompletionCache.LoadFromFile(GetTabCompletionCacheFilename()) cached_results = cache.GetCachedResults(prefix) timing_log_entry_type = '' if cached_results is not None: results = cached_results timing_log_entry_type = ' (from cache)' else: try: results = self._PerformCloudListing(wildcard_url, timeout) if self._bucket_only and len(results) == 1: results = [StripOneSlash(results[0])] partial_results = (len(results) == _TAB_COMPLETE_MAX_RESULTS) cache.UpdateCache(prefix, results, partial_results) except TimeoutError: timing_log_entry_type = ' (request timeout)' results = [] cache.WriteToFile(GetTabCompletionCacheFilename()) end_time = time.time() num_results = len(results) elapsed_seconds = end_time - start_time _WriteTimingLog( '%s results%s in %.2fs, %.2f results/second for prefix: %s\n' % (num_results, timing_log_entry_type, elapsed_seconds, num_results / elapsed_seconds, prefix)) return results class CloudOrLocalObjectCompleter(object): """Completer object for Cloud URLs or local files. Invokes the Cloud object completer if the input looks like a Cloud URL and falls back to local file completer otherwise. """ def __init__(self, gsutil_api): self.cloud_object_completer = CloudObjectCompleter(gsutil_api) self.local_object_completer = LocalObjectCompleter() def __call__(self, prefix, kwargs): if IsFileUrlString(prefix): completer = self.local_object_completer else: completer = self.cloud_object_completer return completer(prefix, kwargs) class NoOpCompleter(object): """Completer that always returns 0 results.""" def __call__(self, unused_prefix, **unused_kwargs): return [] def MakeCompleter(completer_type, gsutil_api): """Create a completer instance of the given type. Args: completer_type: The type of completer to create. gsutil_api: gsutil Cloud API instance to use. Returns: A completer instance. Raises: RuntimeError: if completer type is not supported. """ if completer_type == CompleterType.CLOUD_OR_LOCAL_OBJECT: return CloudOrLocalObjectCompleter(gsutil_api) elif completer_type == CompleterType.LOCAL_OBJECT: return LocalObjectCompleter() elif completer_type == CompleterType.LOCAL_OBJECT_OR_CANNED_ACL: return LocalObjectOrCannedACLCompleter() elif completer_type == CompleterType.CLOUD_BUCKET: return CloudObjectCompleter(gsutil_api, bucket_only=True) elif completer_type == CompleterType.CLOUD_OBJECT: return CloudObjectCompleter(gsutil_api) elif completer_type == CompleterType.NO_OP: return NoOpCompleter() else: raise RuntimeError('Unknown completer "%s"' % completer_type) def _WriteTimingLog(message): """Write an entry to the tab completion timing log, if it's enabled.""" if boto.config.getbool('GSUtil', 'tab_completion_time_logs', False): with open(GetTabCompletionLogFilename(), 'ab') as fp: fp.write(message)	10.77 KB	2025-06-25 08:19
📁 tests	-	2025-06-25 08:26
📁 third_party	-	2025-06-25 08:26
📄 thread_message.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Thread message classes. Messages are added to the status queue. """ from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import os import threading from apitools.base.py.exceptions import Error as apitools_service_error from six.moves.http_client import error as six_service_error class StatusMessage(object): """General StatusMessage class. All Message classes inherit this StatusMessage class. """ def __init__(self, message_time, process_id=None, thread_id=None): """Creates a Message. Args: message_time: Time that this message was created (since Epoch). process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ self.time = message_time self.process_id = process_id or os.getpid() self.thread_id = thread_id or threading.current_thread().ident def __str__(self): """Returns a string with a valid constructor for this message.""" return ( '%s(%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.time, self.process_id, self.thread_id)) class RetryableErrorMessage(StatusMessage): """Message class for retryable errors encountered by the JSON API. This class contains information about the retryable error encountered to report to analytics collection and to display in the UI. """ def __init__(self, exception, message_time, num_retries=0, total_wait_sec=0, process_id=None, thread_id=None): """Creates a RetryableErrorMessage. Args: exception: The retryable error that was thrown. message_time: Float representing when message was created (seconds since Epoch). num_retries: The number of retries consumed so far. total_wait_sec: The total amount of time waited so far in retrying. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(RetryableErrorMessage, self).__init__(message_time, process_id=process_id, thread_id=thread_id) self.error_type = exception.__class__.__name__ # The socket module error class names aren't descriptive enough, so we # make the error_type more specific. Standard Python uses the module name # 'socket' while PyPy uses '_socket' instead. if exception.__class__.__module__ in ('socket', '_socket'): self.error_type = 'Socket' + exception.__class__.__name__.capitalize() if (isinstance(exception, apitools_service_error) or isinstance(exception, six_service_error)): self.is_service_error = True else: self.is_service_error = False # The number of retries consumed to display to the user. self.num_retries = num_retries # The total amount of time waited on the request to display to the user. self.total_wait_sec = total_wait_sec def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s(), num_retries=%s, total_wait_sec=%s, ' 'time=%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.error_type, self.num_retries, self.total_wait_sec, self.time, self.process_id, self.thread_id)) class FinalMessage(StatusMessage): """Creates a FinalMessage. A FinalMessage simply indicates that we have finished our operation. """ def __init__(self, message_time): """Creates a FinalMessage. Args: message_time: Float representing when message was created (seconds since Epoch). """ super(FinalMessage, self).__init__(message_time) class MetadataMessage(StatusMessage): """Creates a MetadataMessage. A MetadataMessage simply indicates that a metadata operation on a given object has been successfully done. The only passed argument is the time when such operation has finished. """ def __init__(self, message_time): """Creates a MetadataMessage. Args: message_time: Float representing when message was created (seconds since Epoch). """ super(MetadataMessage, self).__init__(message_time) class FileMessage(StatusMessage): """Marks start or end of an operation for a file, cloud object or component. This class contains general information about each file/component. With that, information such as total size and estimated time remaining may be calculated beforehand. """ # Enum message types FILE_DOWNLOAD = 1 FILE_UPLOAD = 2 FILE_CLOUD_COPY = 3 FILE_LOCAL_COPY = 4 FILE_DAISY_COPY = 5 FILE_REWRITE = 6 FILE_HASH = 7 COMPONENT_TO_UPLOAD = 8 # EXISTING_COMPONENT describes a component that already exists. The field # finished does not apply quite well for it, but the convention used by the UI # is to process the component alongside FileMessages from other components # when finished==False, and to ignore a FileMessage made for # EXISTING_COMPONENT when finished==True. EXISTING_COMPONENT = 9 COMPONENT_TO_DOWNLOAD = 10 EXISTING_OBJECT_TO_DELETE = 11 def __init__(self, src_url, dst_url, message_time, size=None, finished=False, component_num=None, message_type=None, bytes_already_downloaded=None, process_id=None, thread_id=None): """Creates a FileMessage. Args: src_url: FileUrl/CloudUrl representing the source file. dst_url: FileUrl/CloudUrl representing the destination file. message_time: Float representing when message was created (seconds since Epoch). size: Total size of this file/component, in bytes. finished: Boolean to indicate whether this is starting or finishing a file/component transfer. component_num: Component number, if dealing with a component. message_type: Type of the file/component. bytes_already_downloaded: Specific field for resuming downloads. When starting a component download, it tells how many bytes were already downloaded. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(FileMessage, self).__init__(message_time, process_id=process_id, thread_id=thread_id) self.src_url = src_url self.dst_url = dst_url self.size = size self.component_num = component_num self.finished = finished self.message_type = message_type self.bytes_already_downloaded = bytes_already_downloaded def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(\'%s\', \'%s\', %s, size=%s, finished=%s, component_num=%s, ' 'message_type=%s, bytes_already_downloaded=%s, process_id=%s, ' 'thread_id=%s)' % (self.__class__.__name__, self.src_url, self.dst_url, self.time, self.size, self.finished, self.component_num, self.message_type, self.bytes_already_downloaded, self.process_id, self.thread_id)) class ProgressMessage(StatusMessage): """Message class for a file/object/component progress. This class contains specific information about the current progress of a file, cloud object or single component. """ def __init__(self, size, processed_bytes, src_url, message_time, dst_url=None, component_num=None, operation_name=None, process_id=None, thread_id=None): """Creates a ProgressMessage. Args: size: Integer for total size of this file/component, in bytes. processed_bytes: Integer for number of bytes already processed from that specific component, which means processed_bytes <= size. src_url: FileUrl/CloudUrl representing the source file. message_time: Float representing when message was created (seconds since Epoch). dst_url: FileUrl/CloudUrl representing the destination file, or None for unary operations like hashing. component_num: Indicates the component number, if any. operation_name: Name of the operation that is being made over that component. process_id: Process ID that produced this message (overridable for testing). thread_id: Thread ID that produced this message (overridable for testing). """ super(ProgressMessage, self).__init__(message_time) self.size = size self.processed_bytes = processed_bytes self.component_num = component_num self.src_url = src_url self.dst_url = dst_url self.finished = (size == processed_bytes) self.operation_name = operation_name def __str__(self): """Returns a string with a valid constructor for this message.""" # For a valid constructor, None should not be quoted. dst_url_string = ('\'%s\'' % self.dst_url) if self.dst_url else None operation_name_string = ( '\'%s\'' % self.operation_name) if self.operation_name else None return ('%s(%s, %s, \'%s\', %s, dst_url=%s, component_num=%s, ' 'operation_name=%s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.size, self.processed_bytes, self.src_url, self.time, dst_url_string, self.component_num, operation_name_string, self.process_id, self.thread_id)) class SeekAheadMessage(StatusMessage): """Message class for results obtained by SeekAheadThread(). It estimates the number of objects and total size in case the task_queue cannot hold all tasks at once (only used in large operations). This class contains information about all the objects yet to be processed. """ def __init__(self, num_objects, size, message_time): """Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the SeekAheadThread estimates. size: Total size corresponding to the sum of the size of objects iterated by SeekAheadThread. message_time: Float representing when message was created (seconds since Epoch). """ super(SeekAheadMessage, self).__init__(message_time) self.num_objects = num_objects self.size = size def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s, %s, process_id=%s, thread_id=%s)' % (self.__class__.__name__, self.num_objects, self.size, self.time, self.process_id, self.thread_id)) class ProducerThreadMessage(StatusMessage): """Message class for results obtained by calculations made on ProducerThread. It estimates the number of objects and total size currently dealty by task_queue. If the task_queue cannot support all objects at once, the SeekAheadThread will be responsible for sending an accurate message. """ def __init__(self, num_objects, size, message_time, finished=False): """Creates a SeekAheadMessage. Args: num_objects: Number of total objects that the task_queue has. size: Total size corresponding to the sum of the size of objects iterated by the task_queue message_time: Float representing when message was created (seconds since Epoch). finished: Boolean to indicate whether this is the final message from the ProducerThread. The difference is that this message displays the correct total size and number of objects, whereas the previous ones were periodic (on the number of files) updates. """ super(ProducerThreadMessage, self).__init__(message_time) self.num_objects = num_objects self.size = size self.finished = finished def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s, %s, finished=%s)' % (self.__class__.__name__, self.num_objects, self.size, self.time, self.finished)) class PerformanceSummaryMessage(StatusMessage): """Message class to log PerformanceSummary parameters. This class acts as a relay between a multiprocess/multithread situation and the global status queue, from which the PerformanceSummary info gets consumed. """ def __init__(self, message_time, uses_slice): """Creates a PerformanceSummaryMessage. Args: message_time: Float representing when message was created (seconds since Epoch). uses_slice: True if the command uses slice parallelism. """ super(PerformanceSummaryMessage, self).__init__(message_time, process_id=None, thread_id=None) self.uses_slice = uses_slice def __str__(self): """Returns a string with a valid constructor for this message.""" return ('%s(%s, %s)' % (self.__class__.__name__, self.time, self.uses_slice))	13.71 KB	2025-06-25 08:19
📄 tracker_file.py # -- coding: utf-8 -- # Copyright 2015 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Helper functions for tracker file functionality.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import errno import hashlib import json import os import re import sys import six from boto import config from gslib.exception import CommandException from gslib.utils.boto_util import GetGsutilStateDir from gslib.utils.boto_util import ResumableThreshold from gslib.utils.constants import UTF8 from gslib.utils.hashing_helper import GetMd5 from gslib.utils.system_util import CreateDirIfNeeded # The maximum length of a file name can vary wildly between different # operating systems, so we always ensure that tracker files are less # than 100 characters in order to avoid any such issues. MAX_TRACKER_FILE_NAME_LENGTH = 100 TRACKER_FILE_UNWRITABLE_EXCEPTION_TEXT = ( 'Couldn\'t write tracker file (%s): %s. This can happen if gsutil is ' 'configured to save tracker files to an unwritable directory)') # Format for upload tracker files. ENCRYPTION_UPLOAD_TRACKER_ENTRY = 'encryption_key_sha256' SERIALIZATION_UPLOAD_TRACKER_ENTRY = 'serialization_data' class TrackerFileType(object): UPLOAD = 'upload' DOWNLOAD = 'download' DOWNLOAD_COMPONENT = 'download_component' PARALLEL_UPLOAD = 'parallel_upload' SLICED_DOWNLOAD = 'sliced_download' REWRITE = 'rewrite' def _HashFilename(filename): """Apply a hash function (SHA1) to shorten the passed file name. The spec for the hashed file name is as follows: TRACKER_<hash>_<trailing> where hash is a SHA1 hash on the original file name and trailing is the last 16 chars from the original file name. Max file name lengths vary by operating system so the goal of this function is to ensure the hashed version takes fewer than 100 characters. Args: filename: file name to be hashed. May be unicode or bytes. Returns: shorter, hashed version of passed file name """ if isinstance(filename, six.text_type): filename_bytes = filename.encode(UTF8) filename_str = filename else: filename_bytes = filename filename_str = filename.decode(UTF8) m = hashlib.sha1(filename_bytes) return 'TRACKER_' + m.hexdigest() + '.' + filename_str[-16:] def CreateTrackerDirIfNeeded(): """Looks up or creates the gsutil tracker file directory. This is the configured directory where gsutil keeps its resumable transfer tracker files. This function creates it if it doesn't already exist. Returns: The pathname to the tracker directory. """ tracker_dir = config.get('GSUtil', 'resumable_tracker_dir', os.path.join(GetGsutilStateDir(), 'tracker-files')) CreateDirIfNeeded(tracker_dir) return tracker_dir def GetRewriteTrackerFilePath(src_bucket_name, src_obj_name, dst_bucket_name, dst_obj_name, api_selector): """Gets the tracker file name described by the arguments. Args: src_bucket_name: Source bucket (string). src_obj_name: Source object (string). dst_bucket_name: Destination bucket (string). dst_obj_name: Destination object (string) api_selector: API to use for this operation. Returns: File path to tracker file. """ # Encode the src and dest bucket and object names into the tracker file # name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'rewrite__%s__%s__%s__%s__%s.token' % (src_bucket_name, src_obj_name, dst_bucket_name, dst_obj_name, api_selector))) return _HashAndReturnPath(res_tracker_file_name, TrackerFileType.REWRITE) def GetTrackerFilePath(dst_url, tracker_file_type, api_selector, src_url=None, component_num=None): """Gets the tracker file name described by the arguments. Args: dst_url: Destination URL for tracker file. tracker_file_type: TrackerFileType for this operation. api_selector: API to use for this operation. src_url: Source URL for the source file name for parallel uploads. component_num: Component number if this is a download component, else None. Returns: File path to tracker file. """ if tracker_file_type == TrackerFileType.UPLOAD: # Encode the dest bucket and object name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_upload__%s__%s__%s.url' % (dst_url.bucket_name, dst_url.object_name, api_selector))) elif tracker_file_type == TrackerFileType.DOWNLOAD: # Encode the fully-qualified dest file name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_download__%s__%s.etag' % (os.path.realpath(dst_url.object_name), api_selector))) elif tracker_file_type == TrackerFileType.DOWNLOAD_COMPONENT: # Encode the fully-qualified dest file name and the component number # into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'resumable_download__%s__%s__%d.etag' % (os.path.realpath(dst_url.object_name), api_selector, component_num))) elif tracker_file_type == TrackerFileType.PARALLEL_UPLOAD: # Encode the dest bucket and object names as well as the source file name # into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'parallel_upload__%s__%s__%s__%s.url' % (dst_url.bucket_name, dst_url.object_name, src_url, api_selector))) elif tracker_file_type == TrackerFileType.SLICED_DOWNLOAD: # Encode the fully-qualified dest file name into the tracker file name. res_tracker_file_name = (re.sub( '[/\\\\]', '_', 'sliced_download__%s__%s.etag' % (os.path.realpath(dst_url.object_name), api_selector))) elif tracker_file_type == TrackerFileType.REWRITE: # Should use GetRewriteTrackerFilePath instead. raise NotImplementedError() return _HashAndReturnPath(res_tracker_file_name, tracker_file_type) def DeleteDownloadTrackerFiles(dst_url, api_selector): """Deletes all tracker files corresponding to an object download. Args: dst_url: StorageUrl describing the destination file. api_selector: The Cloud API implementation used. """ # Delete non-sliced download tracker file. DeleteTrackerFile( GetTrackerFilePath(dst_url, TrackerFileType.DOWNLOAD, api_selector)) # Delete all sliced download tracker files. tracker_files = GetSlicedDownloadTrackerFilePaths(dst_url, api_selector) for tracker_file in tracker_files: DeleteTrackerFile(tracker_file) def GetSlicedDownloadTrackerFilePaths(dst_url, api_selector, num_components=None): """Gets a list of sliced download tracker file paths. The list consists of the parent tracker file path in index 0, and then any existing component tracker files in [1:]. Args: dst_url: Destination URL for tracker file. api_selector: API to use for this operation. num_components: The number of component tracker files, if already known. If not known, the number will be retrieved from the parent tracker file on disk. Returns: File path to tracker file. """ parallel_tracker_file_path = GetTrackerFilePath( dst_url, TrackerFileType.SLICED_DOWNLOAD, api_selector) tracker_file_paths = [parallel_tracker_file_path] # If we don't know the number of components, check the tracker file. if num_components is None: tracker_file = None try: tracker_file = open(parallel_tracker_file_path, 'r') num_components = json.load(tracker_file)['num_components'] except (IOError, ValueError): return tracker_file_paths finally: if tracker_file: tracker_file.close() for i in range(num_components): tracker_file_paths.append( GetTrackerFilePath(dst_url, TrackerFileType.DOWNLOAD_COMPONENT, api_selector, component_num=i)) return tracker_file_paths def _HashAndReturnPath(res_tracker_file_name, tracker_file_type): """Hashes and returns a tracker file path. Args: res_tracker_file_name: The tracker file name prior to it being hashed. tracker_file_type: The TrackerFileType of res_tracker_file_name. Returns: Final (hashed) tracker file path. """ resumable_tracker_dir = CreateTrackerDirIfNeeded() hashed_tracker_file_name = _HashFilename(res_tracker_file_name) tracker_file_name = '%s_%s' % (str(tracker_file_type).lower(), hashed_tracker_file_name) tracker_file_path = '%s%s%s' % (resumable_tracker_dir, os.sep, tracker_file_name) assert len(tracker_file_name) < MAX_TRACKER_FILE_NAME_LENGTH return tracker_file_path def DeleteTrackerFile(tracker_file_name): if tracker_file_name and os.path.exists(tracker_file_name): os.unlink(tracker_file_name) def HashRewriteParameters(src_obj_metadata, dst_obj_metadata, projection, src_generation=None, gen_match=None, meta_gen_match=None, canned_acl=None, max_bytes_per_call=None, src_dec_key_sha256=None, dst_enc_key_sha256=None, fields=None): """Creates an MD5 hex digest of the parameters for a rewrite call. Resuming rewrites requires that the input parameters are identical. Thus, the rewrite tracker file needs to represent the input parameters. For easy comparison, hash the input values. If a user does a performs a same-source/same-destination rewrite via a different command (for example, with a changed ACL), the hashes will not match and we will restart the rewrite from the beginning. Args: src_obj_metadata: apitools Object describing source object. Must include bucket, name, and etag. dst_obj_metadata: apitools Object describing destination object. Must include bucket and object name projection: Projection used for the API call. src_generation: Optional source generation. gen_match: Optional generation precondition. meta_gen_match: Optional metageneration precondition. canned_acl: Optional canned ACL string. max_bytes_per_call: Optional maximum bytes rewritten per call. src_dec_key_sha256: Optional SHA256 hash string of decryption key for source object. dst_enc_key_sha256: Optional SHA256 hash string of encryption key for destination object. fields: Optional fields to include in response to call. Returns: MD5 hex digest Hash of the input parameters, or None if required parameters are missing. """ if (not src_obj_metadata or not src_obj_metadata.bucket or not src_obj_metadata.name or not src_obj_metadata.etag or not dst_obj_metadata or not dst_obj_metadata.bucket or not dst_obj_metadata.name or not projection): return md5_hash = GetMd5() for input_param in ( src_obj_metadata, dst_obj_metadata, projection, src_generation, gen_match, meta_gen_match, canned_acl, fields, max_bytes_per_call, src_dec_key_sha256, dst_enc_key_sha256, ): # Tracker file matching changed between gsutil 4.15 -> 4.16 and will cause # rewrites to start over from the beginning on a gsutil version upgrade. if input_param is not None: md5_hash.update(six.text_type(input_param).encode('UTF8')) return md5_hash.hexdigest() def ReadRewriteTrackerFile(tracker_file_name, rewrite_params_hash): """Attempts to read a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. Returns: String rewrite_token for resuming rewrite requests if a matching tracker file exists, None otherwise (which will result in starting a new rewrite). """ # Check to see if we already have a matching tracker file. tracker_file = None if not rewrite_params_hash: return try: tracker_file = open(tracker_file_name, 'r') existing_hash = tracker_file.readline().rstrip('\n') if existing_hash == rewrite_params_hash: # Next line is the rewrite token. return tracker_file.readline().rstrip('\n') except IOError as e: # Ignore non-existent file (happens first time a rewrite is attempted. if e.errno != errno.ENOENT: sys.stderr.write( ('Couldn\'t read Copy tracker file (%s): %s. Restarting copy ' 'from scratch.' % (tracker_file_name, e.strerror))) finally: if tracker_file: tracker_file.close() def WriteRewriteTrackerFile(tracker_file_name, rewrite_params_hash, rewrite_token): """Writes a rewrite tracker file. Args: tracker_file_name: Tracker file path string. rewrite_params_hash: MD5 hex digest of rewrite call parameters constructed by HashRewriteParameters. rewrite_token: Rewrite token string returned by the service. """ _WriteTrackerFile(tracker_file_name, '%s\n%s\n' % (rewrite_params_hash, rewrite_token)) def ReadOrCreateDownloadTrackerFile(src_obj_metadata, dst_url, logger, api_selector, start_byte, existing_file_size, component_num=None): """Checks for a download tracker file and creates one if it does not exist. The methodology for determining the download start point differs between normal and sliced downloads. For normal downloads, the existing bytes in the file are presumed to be correct and have been previously downloaded from the server (if a tracker file exists). In this case, the existing file size is used to determine the download start point. For sliced downloads, the number of bytes previously retrieved from the server cannot be determined from the existing file size, and so the number of bytes known to have been previously downloaded is retrieved from the tracker file. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. logger: For outputting log messages. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: tracker_file_name: The name of the tracker file, if one was used. download_start_byte: The first byte that still needs to be downloaded. """ assert src_obj_metadata.etag tracker_file_name = None if src_obj_metadata.size < ResumableThreshold(): # Don't create a tracker file for a small downloads; cross-process resumes # won't work, but restarting a small download is inexpensive. return tracker_file_name, start_byte download_name = dst_url.object_name if component_num is None: tracker_file_type = TrackerFileType.DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT download_name += ' component %d' % component_num tracker_file_name = GetTrackerFilePath(dst_url, tracker_file_type, api_selector, component_num=component_num) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = open(tracker_file_name, 'r') if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == src_obj_metadata.etag: return tracker_file_name, existing_file_size elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == src_obj_metadata.etag and component_data['generation'] == src_obj_metadata.generation): return tracker_file_name, component_data['download_start_byte'] logger.warn('Tracker file doesn\'t match for download of %s. Restarting ' 'download from scratch.' % download_name) except (IOError, ValueError) as e: # Ignore non-existent file (happens first time a download # is attempted on an object), but warn user for other errors. if isinstance(e, ValueError) or e.errno != errno.ENOENT: logger.warn('Couldn\'t read download tracker file (%s): %s. Restarting ' 'download from scratch.' % (tracker_file_name, str(e))) finally: if tracker_file: tracker_file.close() # There wasn't a matching tracker file, so create one and then start the # download from scratch. if tracker_file_type is TrackerFileType.DOWNLOAD: _WriteTrackerFile(tracker_file_name, '%s\n' % src_obj_metadata.etag) elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: WriteDownloadComponentTrackerFile(tracker_file_name, src_obj_metadata, start_byte) return tracker_file_name, start_byte def GetDownloadStartByte(src_obj_metadata, dst_url, api_selector, start_byte, existing_file_size, component_num=None): """Returns the download starting point. The methodology of this function is the same as in ReadOrCreateDownloadTrackerFile, with the difference that we are not interested here in possibly creating a tracker file. In case there is no tracker file, this means the download starting point is start_byte. Args: src_obj_metadata: Metadata for the source object. Must include etag and generation. dst_url: Destination URL for tracker file. api_selector: API to use for this operation. start_byte: The start byte of the byte range for this download. existing_file_size: Size of existing file for this download on disk. component_num: The component number, if this is a component of a parallel download, else None. Returns: download_start_byte: The first byte that still needs to be downloaded. """ assert src_obj_metadata.etag tracker_file_name = None if src_obj_metadata.size < ResumableThreshold(): # There is no tracker file for small downloads; this means we start from # scratch. return start_byte if component_num is None: tracker_file_type = TrackerFileType.DOWNLOAD else: tracker_file_type = TrackerFileType.DOWNLOAD_COMPONENT tracker_file_name = GetTrackerFilePath(dst_url, tracker_file_type, api_selector, component_num=component_num) tracker_file = None # Check to see if we already have a matching tracker file. try: tracker_file = open(tracker_file_name, 'r') if tracker_file_type is TrackerFileType.DOWNLOAD: etag_value = tracker_file.readline().rstrip('\n') if etag_value == src_obj_metadata.etag: return existing_file_size elif tracker_file_type is TrackerFileType.DOWNLOAD_COMPONENT: component_data = json.loads(tracker_file.read()) if (component_data['etag'] == src_obj_metadata.etag and component_data['generation'] == src_obj_metadata.generation): return component_data['download_start_byte'] except (IOError, ValueError): # If the file does not exist, there is not much we can do at this point. pass finally: if tracker_file: tracker_file.close() # There wasn't a matching tracker file, which means our starting point is # start_byte. return start_byte def WriteDownloadComponentTrackerFile(tracker_file_name, src_obj_metadata, current_file_pos): """Updates or creates a download component tracker file on disk. Args: tracker_file_name: The name of the tracker file. src_obj_metadata: Metadata for the source object. Must include etag. current_file_pos: The current position in the file. """ component_data = { 'etag': src_obj_metadata.etag, 'generation': src_obj_metadata.generation, 'download_start_byte': current_file_pos, } _WriteTrackerFile(tracker_file_name, json.dumps(component_data)) def _WriteTrackerFile(tracker_file_name, data): """Creates a tracker file, storing the input data.""" try: fd = os.open(tracker_file_name, os.O_WRONLY \| os.O_CREAT \| os.O_TRUNC, 0o600) with os.fdopen(fd, 'w') as tf: tf.write(data) return False except (IOError, OSError) as e: raise RaiseUnwritableTrackerFileException(tracker_file_name, e.strerror) def WriteJsonDataToTrackerFile(tracker_file_name, data): """Create a tracker file and write json data to it. Raises: TypeError: If the data is not JSON serializable """ try: json_str = json.dumps(data) except TypeError as err: raise RaiseUnwritableTrackerFileException(tracker_file_name, err.strerror) _WriteTrackerFile(tracker_file_name, json_str) def GetUploadTrackerData(tracker_file_name, logger, encryption_key_sha256=None): """Reads tracker data from an upload tracker file if it exists. Deletes the tracker file if it uses an old format or the desired encryption key has changed. Args: tracker_file_name: Tracker file name for this upload. logger: logging.Logger for outputting log messages. encryption_key_sha256: Encryption key SHA256 for use in this upload, if any. Returns: Serialization data if the tracker file already exists (resume existing upload), None otherwise. """ tracker_file = None remove_tracker_file = False encryption_restart = False # If we already have a matching tracker file, get the serialization data # so that we can resume the upload. try: tracker_file = open(tracker_file_name, 'r') tracker_data = tracker_file.read() tracker_json = json.loads(tracker_data) if tracker_json[ENCRYPTION_UPLOAD_TRACKER_ENTRY] != encryption_key_sha256: encryption_restart = True remove_tracker_file = True else: return tracker_json[SERIALIZATION_UPLOAD_TRACKER_ENTRY] except IOError as e: # Ignore non-existent file (happens first time a upload is attempted on an # object, or when re-starting an upload after a # ResumableUploadStartOverException), but warn user for other errors. if e.errno != errno.ENOENT: logger.warn( 'Couldn\'t read upload tracker file (%s): %s. Restarting ' 'upload from scratch.', tracker_file_name, e.strerror) except (KeyError, ValueError) as e: # Old tracker files used a non-JSON format; rewrite it and assume no # encryption key. remove_tracker_file = True if encryption_key_sha256 is not None: encryption_restart = True else: # If encryption key is still None, we can resume using the old format. return tracker_data finally: if tracker_file: tracker_file.close() if encryption_restart: logger.warn( 'Upload tracker file (%s) does not match current encryption ' 'key. Restarting upload from scratch with a new tracker ' 'file that uses the current encryption key.', tracker_file_name) if remove_tracker_file: DeleteTrackerFile(tracker_file_name) def RaiseUnwritableTrackerFileException(tracker_file_name, error_str): """Raises an exception when unable to write the tracker file.""" raise CommandException(TRACKER_FILE_UNWRITABLE_EXCEPTION_TEXT % (tracker_file_name, error_str))	24.23 KB	2025-06-25 08:19
📄 tz_utc.py # -- coding: utf-8 -- # Copyright 2018 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Provides a tzinfo subclass to represent the UTC timezone.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from datetime import timedelta from datetime import tzinfo class UTC(tzinfo): """Timezone information class used to convert datetime timestamps to UTC. This class is necessary to convert timestamps to UTC. By default Python datetime objects are timezone unaware. This created problems when interacting with cloud object timestamps which are timezone-aware. This issue appeared when handling the timeCreated metadata attribute; the values returned by the service were placed in RFC 3339 format in the storage_v1_messages module. RFC 3339 requires a timezone in any timestamp. This caused problems as the datetime object elsewhere in the code was timezone unaware and was different by exactly one hour. The main problem was that the local system used daylight savings time which adjusted the timestamp ahead by one hour. """ def utcoffset(self, _): """An offset of the number of minutes away from UTC this tzinfo object is. Returns: A time duration of zero. UTC is zero minutes away from UTC. """ return timedelta(0) def tzname(self, _): """A method to retrieve the name of this timezone object. Returns: The name of the timezone (i.e. 'UTC'). """ return 'UTC' def dst(self, _): """A fixed offset to handle daylight savings time (DST). Returns: A time duration of zero as UTC does not use DST. """ return timedelta(0)	2.2 KB	2025-06-25 08:19
📄 ui_controller.py # -- coding: utf-8 -- # Copyright 2016 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base classes for gsutil UI controller, UIThread and MainThreadUIQueue.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals from collections import deque import sys import threading import time from six.moves import queue as Queue from gslib.metrics import LogPerformanceSummaryParams from gslib.metrics import LogRetryableError from gslib.thread_message import FileMessage from gslib.thread_message import FinalMessage from gslib.thread_message import MetadataMessage from gslib.thread_message import PerformanceSummaryMessage from gslib.thread_message import ProducerThreadMessage from gslib.thread_message import ProgressMessage from gslib.thread_message import RetryableErrorMessage from gslib.thread_message import SeekAheadMessage from gslib.thread_message import StatusMessage from gslib.utils import parallelism_framework_util from gslib.utils.unit_util import DecimalShort from gslib.utils.unit_util import HumanReadableWithDecimalPlaces from gslib.utils.unit_util import MakeHumanReadable from gslib.utils.unit_util import PrettyTime _ZERO_TASKS_TO_DO_ARGUMENT = ( parallelism_framework_util.ZERO_TASKS_TO_DO_ARGUMENT) class EstimationSource(object): """enum for total size source.""" # Integer to indicate total size came from the final ProducerThreadMessage. # It has priority over all other total_size sources. PRODUCER_THREAD_FINAL = 1 # Integer to indicate total size came from SeekAheadThread. # It has priority over self.SEEK_AHEAD_THREAD and over # self.INDIVIDUAL_MESSAGES. SEEK_AHEAD_THREAD = 2 # Integer to indicate total size came from a ProducerThread estimation. # It has priority over self.INDIVIDUAL_MESSAGES. PRODUCER_THREAD_ESTIMATE = 3 # Stores the actual source from total_size. We start from FileMessages or # MetadataMessages. INDIVIDUAL_MESSAGES = 4 # Note: this priority based model was used in case we add new sources for # total_size in the future. It also allows us to search for smaller numbers # (larger priorities) rather than having to list those with higher priority. def BytesToFixedWidthString(num_bytes, decimal_places=1): """Adjusts proper width for printing num_bytes in readable format. Args: num_bytes: The number of bytes we must display. decimal_places: The standard number of decimal places. Returns: String of fixed width representing num_bytes. """ human_readable = HumanReadableWithDecimalPlaces(num_bytes, decimal_places=decimal_places) number_format = human_readable.split() if int(round(float(number_format[0]))) >= 1000: # If we are in the [1000:1024) range for the whole part of the number, # we must remove the decimal part. last_character = len(number_format[0]) - decimal_places - 1 number_format[0] = number_format[0][:last_character] return '%9s' % (' '.join(number_format)) class StatusMessageManager(object): """General manager for common functions shared by data and metadata managers. This subclass has the responsibility of having a common constructor and the same handler for SeekAheadMessages and ProducerThreadMessages. """ class _ThroughputInformation(object): """Class that contains all information needed for throughput calculation. This _ThroughputInformation is used to track progress and time at several points of our operation. """ def __init__(self, progress, report_time): """Constructor of _ThroughputInformation. Args: progress: The current progress, in bytes/second or objects/second. report_time: Float representing when progress was reported (seconds since Epoch). """ self.progress = progress self.time = report_time def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=80): """Instantiates a StatusMessageManager. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field, but it will affect throughput and time remaining estimations. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. console_width: Width to display on console. This should not adjust the visual output, just the space padding. For proper visualization, we recommend setting this field to at least 80. """ self.update_message_period = update_message_period self.update_spinner_period = update_spinner_period self.sliding_throughput_period = sliding_throughput_period self.first_throughput_latency = first_throughput_latency self.quiet_mode = quiet_mode self.custom_time = custom_time self.verbose = verbose self.console_width = console_width # Initial estimation source for number of objects and total size # is through individual FileMessages or individual MetadataMessages, # depending on the StatusMessageManager superclass. self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.total_size_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects = 0 # Only used on data operations. Will remain 0 for metadata operations. self.total_size = 0 # Time at last info update displayed. self.refresh_message_time = (self.custom_time if self.custom_time else time.time()) self.start_time = self.refresh_message_time # Time at last spinner update. self.refresh_spinner_time = self.refresh_message_time # Measured in objects/second or bytes/second, depending on the superclass. self.throughput = 0.0 # Deque of _ThroughputInformation to help with throughput calculation. self.old_progress = deque() self.last_progress_time = 0 self.spinner_char_list = ['/', '-', '\\', '\|'] self.current_spinner_index = 0 self.objects_finished = 0 self.num_objects = 0 # Number of objects being processed # This overrides time constraints for updating and displaying # important information, such as having finished to process an object. self.object_report_change = False self.final_message = False def GetSpinner(self): """Returns the current spinner character. Returns: char_to_print: Char to be printed as the spinner """ return self.spinner_char_list[self.current_spinner_index] def UpdateSpinner(self): """Updates the current spinner character.""" self.current_spinner_index = ((self.current_spinner_index + 1) % len(self.spinner_char_list)) def _HandleProducerThreadMessage(self, status_message): """Handles a ProducerThreadMessage. Args: status_message: The ProducerThreadMessage to be processed. """ if status_message.finished: # This means this was a final ProducerThreadMessage. if self.num_objects_source >= EstimationSource.PRODUCER_THREAD_FINAL: self.num_objects_source = EstimationSource.PRODUCER_THREAD_FINAL self.num_objects = status_message.num_objects if (self.total_size_source >= EstimationSource.PRODUCER_THREAD_FINAL and status_message.size): self.total_size_source = EstimationSource.PRODUCER_THREAD_FINAL self.total_size = status_message.size return if self.num_objects_source >= EstimationSource.PRODUCER_THREAD_ESTIMATE: self.num_objects_source = EstimationSource.PRODUCER_THREAD_ESTIMATE self.num_objects = status_message.num_objects if (self.total_size_source >= EstimationSource.PRODUCER_THREAD_ESTIMATE and status_message.size): self.total_size_source = EstimationSource.PRODUCER_THREAD_ESTIMATE self.total_size = status_message.size def _HandleSeekAheadMessage(self, status_message, stream): """Handles a SeekAheadMessage. Args: status_message: The SeekAheadMessage to be processed. stream: Stream to print messages. """ estimate_message = ('Estimated work for this command: objects: %s' % status_message.num_objects) if status_message.size: estimate_message += (', total size: %s' % MakeHumanReadable(status_message.size)) if self.total_size_source >= EstimationSource.SEEK_AHEAD_THREAD: self.total_size_source = EstimationSource.SEEK_AHEAD_THREAD self.total_size = status_message.size if self.num_objects_source >= EstimationSource.SEEK_AHEAD_THREAD: self.num_objects_source = EstimationSource.SEEK_AHEAD_THREAD self.num_objects = status_message.num_objects estimate_message += '\n' if not self.quiet_mode: stream.write(estimate_message) def _HandlePerformanceSummaryMessage(self, status_message): """Handles a PerformanceSummaryMessage. Args: status_message: The PerformanceSummaryMessage to be processed. """ LogPerformanceSummaryParams(uses_slice=status_message.uses_slice) def ShouldTrackThroughput(self, cur_time): """Decides whether enough time has passed to start tracking throughput. Args: cur_time: current time. Returns: Whether or not we should track the throughput. """ return cur_time - self.start_time >= self.first_throughput_latency def ShouldPrintProgress(self, cur_time): """Decides whether or not it is time for printing a new progress. Args: cur_time: current time. Returns: Whether or not we should print the progress. """ sufficient_time_elapsed = (cur_time - self.refresh_message_time >= self.update_message_period) # Don't report if we aren't actually going to do anything (for example, # an rsync that will sync 0 objects). nonzero_report = self.num_objects return (sufficient_time_elapsed or self.object_report_change) and (nonzero_report) def ShouldPrintSpinner(self, cur_time): """Decides whether or not it is time for updating the spinner character. Args: cur_time: Current time. Returns: Whether or not we should update and print the spinner. """ return (cur_time - self.refresh_spinner_time > self.update_spinner_period and self.total_size) def PrintSpinner(self, stream=sys.stderr): """Prints a spinner character. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ self.UpdateSpinner() if not self.quiet_mode: stream.write(self.GetSpinner() + '\r') def UpdateThroughput(self, cur_time, cur_progress): """Updates throughput if the required period for calculation has passed. The throughput is calculated by taking all the progress (objects or bytes) processed within the last sliding_throughput_period seconds, and dividing that by the time period between the oldest progress time within that range and the last progress measurement, which are defined by oldest_progress[1] and last_progress_time, respectively. Among the pros of this approach, a connection break or a sudden change in throughput is quickly noticeable. Furthermore, using the last throughput measurement rather than the current time allows us to have a better estimation of the actual throughput. Args: cur_time: Current time to check whether or not it is time for a new throughput measurement. cur_progress: The current progress, in number of objects finished or in bytes. """ while (len(self.old_progress) > 1 and cur_time - self.old_progress[0].time > self.sliding_throughput_period): self.old_progress.popleft() if not self.old_progress: return oldest_progress = self.old_progress[0] if self.last_progress_time == oldest_progress.time: self.throughput = 0 return # If old-progress is not empty and the time of oldest_progress does not # match the last_progress_time, we can safely calculate the throughput. self.throughput = ((cur_progress - oldest_progress.progress) / (self.last_progress_time - oldest_progress.time)) # Just to avoid -0.00 B/s. self.throughput = max(0, self.throughput) def PrintFinalSummaryMessage(self, stream=sys.stderr): """Prints a final message to indicate operation succeeded. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ string_to_print = ('Operation completed over %s objects' % DecimalShort(self.num_objects)) if self.total_size: string_to_print += ('/%s' % HumanReadableWithDecimalPlaces(self.total_size)) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(('\n' + string_to_print + '.' + (max(remaining_width, 0) * ' ') + '\n')) class MetadataManager(StatusMessageManager): """Manages shared state for metadata operations. This manager is specific for metadata operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. """ def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=80): # pylint: disable=g-doc-args """Instantiates a MetadataManager. See argument documentation in StatusMessageManager base class. """ # pylint: enable=g-doc-args super(MetadataManager, self).__init__(update_message_period=update_message_period, update_spinner_period=update_spinner_period, sliding_throughput_period=sliding_throughput_period, first_throughput_latency=first_throughput_latency, quiet_mode=quiet_mode, custom_time=custom_time, verbose=verbose, console_width=console_width) def GetProgress(self): """Gets the progress for a MetadataManager. Returns: The number of finished objects. """ return self.objects_finished def _HandleMetadataMessage(self, status_message): """Handles a MetadataMessage. Args: status_message: The MetadataMessage to be processed. """ self.objects_finished += 1 if self.num_objects_source >= EstimationSource.INDIVIDUAL_MESSAGES: self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects += 1 # Ensures we print periodic progress, and that we send a final message. self.object_report_change = True self.last_progress_time = status_message.time if (self.objects_finished == self.num_objects and self.num_objects_source == EstimationSource.PRODUCER_THREAD_FINAL): self.final_message = True def ProcessMessage(self, status_message, stream): """Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. """ self.object_report_change = False if isinstance(status_message, SeekAheadMessage): self._HandleSeekAheadMessage(status_message, stream) elif isinstance(status_message, ProducerThreadMessage): self._HandleProducerThreadMessage(status_message) elif isinstance(status_message, MetadataMessage): self._HandleMetadataMessage(status_message) elif isinstance(status_message, RetryableErrorMessage): LogRetryableError(status_message) elif isinstance(status_message, PerformanceSummaryMessage): self._HandlePerformanceSummaryMessage(status_message) self.old_progress.append( self._ThroughputInformation(self.objects_finished, status_message.time)) def PrintProgress(self, stream=sys.stderr): """Prints progress and throughput/time estimation. Prints total number of objects and number of finished objects with the percentage of work done, potentially including the throughput (in objects/second) and estimated time remaining. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ # Time to update all information total_remaining = self.num_objects - self.objects_finished if self.throughput: time_remaining = total_remaining / self.throughput else: time_remaining = None char_to_print = self.GetSpinner() if self.num_objects_source <= EstimationSource.SEEK_AHEAD_THREAD: # An example of objects_completed here would be ' [2/3 objects]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + '/' + DecimalShort(self.num_objects) + ' objects]') if self.num_objects == self.objects_finished: percentage = '100' else: percentage = ( '%3d' % min(99, int(100 * float(self.objects_finished) / self.num_objects))) percentage_completed = percentage + '% Done' else: # An example of objects_completed here would be ' [2 objects]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + ' objects]') percentage_completed = '' if (self.refresh_message_time - self.start_time > self.first_throughput_latency): # Should also include throughput. # An example of throughput here would be '2 objects/s' throughput = '%.2f objects/s' % self.throughput if (self.num_objects_source <= EstimationSource.PRODUCER_THREAD_ESTIMATE and self.throughput): # Should also include time remaining. # An example of time remaining would be ' ETA 00:00:11'. time_remaining_str = 'ETA ' + PrettyTime(time_remaining) else: time_remaining_str = '' else: throughput = '' time_remaining_str = '' format_str = ('{char_to_print} {objects_completed} {percentage_completed}' ' {throughput} {time_remaining_str}') string_to_print = format_str.format( char_to_print=char_to_print, objects_completed=objects_completed, percentage_completed=percentage_completed, throughput=throughput, time_remaining_str=time_remaining_str) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(string_to_print + (max(remaining_width, 0) * ' ') + '\r') def CanHandleMessage(self, status_message): """Determines whether this manager is suitable for handling status_message. Args: status_message: The StatusMessage object to be analyzed. Returns: True if this message can be properly handled by this manager, False otherwise. """ if isinstance( status_message, (SeekAheadMessage, ProducerThreadMessage, MetadataMessage, FinalMessage, RetryableErrorMessage, PerformanceSummaryMessage)): return True return False class DataManager(StatusMessageManager): """Manages shared state for data operations. This manager is specific for data operations. Among its main functions, it receives incoming StatusMessages, storing all necessary data about the current and past states of the system necessary to display to the UI. It also provides methods for calculating metrics such as throughput and estimated time remaining. Finally, it provides methods for displaying messages to the UI. """ class _ProgressInformation(object): """Class that contains all progress information needed for a given file. This _ProgressInformation is used as the value associated with a file_name in the dict that stores the information about all processed files. """ def __init__(self, size): """Constructor of _ProgressInformation. Args: size: The total size of the file. """ # Sum of all progress obtained in this operation. self.new_progress_sum = 0 # Sum of all progress from previous operations (mainly for resuming # uploads or resuming downloads). self.existing_progress_sum = 0 # Dict for tracking the progress for each individual component. Key is # of the form (component_num, dst_url) and correspondent element is a # tuple which stores the current progress obtained from this operation, # and the progress obtained from previous operations. self.dict = {} # The total size for the file self.size = size def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, console_width=None): # pylint: disable=g-doc-args """Instantiates a DataManager. See argument documentation in StatusMessageManager base class. """ # pylint: disable=g-doc-args super(DataManager, self).__init__(update_message_period=update_message_period, update_spinner_period=update_spinner_period, sliding_throughput_period=sliding_throughput_period, first_throughput_latency=first_throughput_latency, quiet_mode=quiet_mode, custom_time=custom_time, verbose=verbose, console_width=console_width) self.first_item = True self.total_progress = 0 # Sum of progress for all threads. self.new_progress = 0 self.existing_progress = 0 # Dict containing individual progress for each file. Key is filename # (from src_url). It maps to a _ProgressInformation object. self.individual_file_progress = {} self.component_total = 0 self.finished_components = 0 self.existing_components = 0 def GetProgress(self): """Gets the progress for a DataManager. Returns: The number of processed bytes in this operation. """ return self.new_progress def _HandleFileDescription(self, status_message): """Handles a FileMessage that describes a file. Args: status_message: the FileMessage to be processed. """ if not status_message.finished: # File started. if self.first_item and not self.custom_time: # Set initial time. self.refresh_message_time = status_message.time self.start_time = self.refresh_message_time self.last_throughput_time = self.refresh_message_time self.first_item = False # Gets file name (from src_url). file_name = status_message.src_url.url_string status_message.size = status_message.size if status_message.size else 0 # Creates a new entry on individual_file_progress. self.individual_file_progress[file_name] = self._ProgressInformation( status_message.size) if self.num_objects_source >= EstimationSource.INDIVIDUAL_MESSAGES: # This ensures the file has not been counted on SeekAheadThread or # in ProducerThread. self.num_objects_source = EstimationSource.INDIVIDUAL_MESSAGES self.num_objects += 1 if self.total_size_source >= EstimationSource.INDIVIDUAL_MESSAGES: # This ensures the file size has not been counted on SeekAheadThread or # in ProducerThread. self.total_size_source = EstimationSource.INDIVIDUAL_MESSAGES self.total_size += status_message.size self.object_report_change = True else: # File finished. self.objects_finished += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] total_bytes_transferred = (file_progress.new_progress_sum + file_progress.existing_progress_sum) # Ensures total_progress has the right value. self.total_progress += file_progress.size - total_bytes_transferred self.new_progress += file_progress.size - total_bytes_transferred self.last_progress_time = status_message.time # Deleting _ProgressInformation object to save memory. del self.individual_file_progress[file_name] self.object_report_change = True if (self.objects_finished == self.num_objects and self.num_objects_source == EstimationSource.PRODUCER_THREAD_FINAL): self.final_message = True def _IsFile(self, file_message): """Tells whether or not this FileMessage represent a file. This is needed because FileMessage is used by both files and components. Args: file_message: The FileMessage to be analyzed. Returns: Whether or not this represents a file. """ message_type = file_message.message_type return (message_type == FileMessage.FILE_DOWNLOAD or message_type == FileMessage.FILE_UPLOAD or message_type == FileMessage.FILE_CLOUD_COPY or message_type == FileMessage.FILE_DAISY_COPY or message_type == FileMessage.FILE_LOCAL_COPY or message_type == FileMessage.FILE_REWRITE or message_type == FileMessage.FILE_HASH) def _HandleComponentDescription(self, status_message): """Handles a FileMessage that describes a component. Args: status_message: The FileMessage to be processed. """ if (status_message.message_type == FileMessage.EXISTING_COMPONENT and not status_message.finished): # Existing component: have to ensure total_progress accounts for it. self.existing_components += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] key = (status_message.component_num, status_message.dst_url) file_progress.dict[key] = (0, status_message.size) file_progress.existing_progress_sum += status_message.size self.total_progress += status_message.size self.existing_progress += status_message.size elif ((status_message.message_type == FileMessage.COMPONENT_TO_UPLOAD or status_message.message_type == FileMessage.COMPONENT_TO_DOWNLOAD)): if not status_message.finished: # Component started. self.component_total += 1 if status_message.message_type == FileMessage.COMPONENT_TO_DOWNLOAD: file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] file_progress.existing_progress_sum += ( status_message.bytes_already_downloaded) key = (status_message.component_num, status_message.dst_url) file_progress.dict[key] = (0, status_message.bytes_already_downloaded) self.total_progress += status_message.bytes_already_downloaded self.existing_progress += status_message.bytes_already_downloaded else: # Component finished. self.finished_components += 1 file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] key = (status_message.component_num, status_message.dst_url) last_update = (file_progress.dict[key] if key in file_progress.dict else (0, 0)) self.total_progress += status_message.size - sum(last_update) self.new_progress += status_message.size - sum(last_update) self.last_progress_time = status_message.time file_progress.new_progress_sum += (status_message.size - sum(last_update)) file_progress.dict[key] = (status_message.size - last_update[1], last_update[1]) def _HandleProgressMessage(self, status_message): """Handles a ProgressMessage that tracks progress of a file or component. Args: status_message: The ProgressMessage to be processed. """ # Retrieving index and dict for this file. file_name = status_message.src_url.url_string file_progress = self.individual_file_progress[file_name] # Retrieves last update ((0,0) if no previous update) for this file or # component. To ensure uniqueness (among components), # we use a (component_num, dst_url) tuple as our key. key = (status_message.component_num, status_message.dst_url) last_update = (file_progress.dict[key] if key in file_progress.dict else (0, 0)) status_message.processed_bytes -= last_update[1] file_progress.new_progress_sum += (status_message.processed_bytes - last_update[0]) # Updates total progress with new update from component. self.total_progress += status_message.processed_bytes - last_update[0] self.new_progress += status_message.processed_bytes - last_update[0] # Updates file_progress.dict on component's key. file_progress.dict[key] = (status_message.processed_bytes, last_update[1]) self.last_progress_time = status_message.time def ProcessMessage(self, status_message, stream): """Processes a message from _MainThreadUIQueue or _UIThread. Args: status_message: The StatusMessage item to be processed. stream: Stream to print messages. Here only for SeekAheadThread """ self.object_report_change = False if isinstance(status_message, ProducerThreadMessage): # ProducerThread info. self._HandleProducerThreadMessage(status_message) elif isinstance(status_message, SeekAheadMessage): # SeekAheadThread info. self._HandleSeekAheadMessage(status_message, stream) elif isinstance(status_message, FileMessage): if self._IsFile(status_message): # File info. self._HandleFileDescription(status_message) else: # Component info. self._HandleComponentDescription(status_message) LogPerformanceSummaryParams(file_message=status_message) elif isinstance(status_message, ProgressMessage): # Progress info. self._HandleProgressMessage(status_message) elif isinstance(status_message, RetryableErrorMessage): LogRetryableError(status_message) elif isinstance(status_message, PerformanceSummaryMessage): self._HandlePerformanceSummaryMessage(status_message) self.old_progress.append( self._ThroughputInformation(self.new_progress, status_message.time)) def PrintProgress(self, stream=sys.stderr): """Prints progress and throughput/time estimation. If a ProducerThreadMessage or SeekAheadMessage has been provided, it outputs the number of files completed, number of total files, the current progress, the total size, and the percentage it represents. If none of those have been provided, it only includes the number of files completed, the current progress and total size (which might be updated), with no percentage as we do not know if more files are coming. It may also include time estimation (available only given ProducerThreadMessage or SeekAheadMessage provided) and throughput. For that to happen, there is an extra condition of at least first_throughput_latency seconds having been passed since the UIController started, and that either the ProducerThread or the SeekAheadThread have estimated total number of files and total size. Args: stream: Stream to print messages. Usually sys.stderr, but customizable for testing. """ # Time to update all information. total_remaining = self.total_size - self.total_progress if self.throughput: time_remaining = total_remaining / self.throughput else: time_remaining = None char_to_print = self.GetSpinner() if self.num_objects_source <= EstimationSource.SEEK_AHEAD_THREAD: # An example of objects_completed here would be ' [2/3 files]'. objects_completed = ('[' + DecimalShort(self.objects_finished) + '/' + DecimalShort(self.num_objects) + ' files]') else: # An example of objects_completed here would be ' [2 files]'. objects_completed = '[' + DecimalShort(self.objects_finished) + ' files]' # An example of bytes_progress would be '[101.0 MiB/1.0 GiB]'. bytes_progress = ('[%s/%s]' % (BytesToFixedWidthString( self.total_progress), BytesToFixedWidthString(self.total_size))) if self.total_size_source <= EstimationSource.SEEK_AHEAD_THREAD: if self.num_objects == self.objects_finished: percentage = '100' else: percentage = ( '%3d' % min(99, int(100 * float(self.total_progress) / self.total_size))) percentage_completed = percentage + '% Done' else: percentage_completed = '' if (self.refresh_message_time - self.start_time > self.first_throughput_latency): # Should also include throughput. # An example of throughput here would be ' 82.3 MiB/s' throughput = BytesToFixedWidthString(self.throughput) + '/s' if (self.total_size_source <= EstimationSource.PRODUCER_THREAD_ESTIMATE and self.throughput): # Should also include time remaining. # An example of time remaining would be ' ETA 00:00:11'. time_remaining_str = 'ETA ' + PrettyTime(time_remaining) else: time_remaining_str = '' else: throughput = '' time_remaining_str = '' format_str = ('{char_to_print} {objects_completed}{bytes_progress}' ' {percentage_completed} {throughput} {time_remaining_str}') string_to_print = format_str.format( char_to_print=char_to_print, objects_completed=objects_completed, bytes_progress=bytes_progress, percentage_completed=percentage_completed, throughput=throughput, time_remaining_str=time_remaining_str) remaining_width = self.console_width - len(string_to_print) if not self.quiet_mode: stream.write(string_to_print + (max(remaining_width, 0) * ' ') + '\r') def CanHandleMessage(self, status_message): """Determines whether this manager is suitable for handling status_message. Args: status_message: The StatusMessage object to be analyzed. Returns: True if this message can be properly handled by this manager, False otherwise. """ if isinstance(status_message, ( SeekAheadMessage, ProducerThreadMessage, FileMessage, ProgressMessage, FinalMessage, RetryableErrorMessage, PerformanceSummaryMessage, )): return True return False class UIController(object): """Controller UI class to integrate _MainThreadUIQueue and _UIThread. This class receives messages from _MainThreadUIQueue and _UIThread and send them to an appropriate manager, which will then processes and store data about them. """ def __init__(self, update_message_period=1, update_spinner_period=0.6, sliding_throughput_period=5, first_throughput_latency=10, quiet_mode=False, custom_time=None, verbose=False, dump_status_messages_file=None): """Instantiates a UIController. Args: update_message_period: Minimum period for refreshing and displaying new information. A non-positive value will ignore any time restrictions imposed by this field. update_spinner_period: Minimum period for refreshing and displaying the spinner. A non-positive value will ignore any time restrictions imposed by this field. sliding_throughput_period: Sliding period for throughput calculation. A non-positive value will make it impossible to calculate the throughput. first_throughput_latency: Minimum waiting time before actually displaying throughput info. A non-positive value will ignore any time restrictions imposed by this field. quiet_mode: If True, do not print status messages (but still process them for analytics reporting as necessary). custom_time: If a custom start_time is desired. Used for testing. verbose: Tells whether or not the operation is on verbose mode. dump_status_messages_file: File path for logging all received status messages, for debugging purposes. """ self.verbose = verbose self.update_message_period = update_message_period self.update_spinner_period = update_spinner_period self.sliding_throughput_period = sliding_throughput_period self.first_throughput_latency = first_throughput_latency self.manager = None self.quiet_mode = quiet_mode self.custom_time = custom_time self.console_width = 80 # Console width. Passed to manager. # List storing all estimation messages from SeekAheadThread or # ProducerThread. This is used when we still do not know which manager to # use. self.early_estimation_messages = [] self.printed_final_message = False self.dump_status_message_fp = None if dump_status_messages_file: self.dump_status_message_fp = open(dump_status_messages_file, 'ab') def _HandleMessage(self, status_message, stream, cur_time=None): """Processes a message, updates throughput and prints progress. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. """ self.manager.ProcessMessage(status_message, stream) if self.manager.ShouldPrintProgress(cur_time): if self.manager.ShouldTrackThroughput(cur_time): self.manager.UpdateThroughput(cur_time, self.manager.GetProgress()) self.manager.PrintProgress(stream) self.manager.refresh_message_time = cur_time if self.manager.ShouldPrintSpinner(cur_time): self.manager.PrintSpinner(stream) self.manager.refresh_spinner_time = cur_time if ((isinstance(status_message, FinalMessage) or self.manager.final_message) and self.manager.num_objects and not self.printed_final_message): self.printed_final_message = True LogPerformanceSummaryParams( num_objects_transferred=self.manager.num_objects) self.manager.PrintFinalSummaryMessage(stream) def Call(self, status_message, stream, cur_time=None): """Coordinates UI manager and calls appropriate function to handle message. Args: status_message: Message to be processed. Could be None if UIThread cannot retrieve message from status_queue. stream: Stream to print messages. Usually sys.stderr, but customizable for testing. cur_time: Message time. Used to determine if it is time to refresh output, or calculate throughput. """ if not isinstance(status_message, StatusMessage): if status_message == _ZERO_TASKS_TO_DO_ARGUMENT and not self.manager: # Create a manager to handle early estimation messages before returning. self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time=estimation_message.time) return if self.dump_status_message_fp: # TODO: Add Unicode support to string methods on message classes. # Currently, dump will fail with a UnicodeEncodeErorr if the message # class contains a Unicode attribute. self.dump_status_message_fp.write(str(status_message)) self.dump_status_message_fp.write('\n') if not cur_time: cur_time = status_message.time if not self.manager: if (isinstance(status_message, SeekAheadMessage) or isinstance(status_message, ProducerThreadMessage)): self.early_estimation_messages.append(status_message) return elif isinstance(status_message, MetadataMessage): self.manager = (MetadataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) else: self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, quiet_mode=self.quiet_mode, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) if not self.manager.CanHandleMessage(status_message): if (isinstance(status_message, FileMessage) or isinstance(status_message, ProgressMessage)): # We have to create a DataManager to handle this data message. This is # to avoid a possible race condition where MetadataMessages are sent # before data messages. As such, this means that the DataManager has # priority, and whenever a data message is received, we ignore the # MetadataManager if one exists, and start a DataManager from scratch. # This can be done because we do not need any MetadataMessages to # properly handle a data operation. It could be useful to send the # early estimation messages, if those are available. self.manager = (DataManager( update_message_period=self.update_message_period, update_spinner_period=self.update_spinner_period, sliding_throughput_period=self.sliding_throughput_period, first_throughput_latency=self.first_throughput_latency, custom_time=self.custom_time, verbose=self.verbose, console_width=self.console_width)) for estimation_message in self.early_estimation_messages: self._HandleMessage(estimation_message, stream, cur_time) else: # No need to handle this message. return self._HandleMessage(status_message, stream, cur_time) class MainThreadUIQueue(object): """Handles status display and processing in the main thread / master process. This class emulates a queue to cover main-thread activity before or after Apply, as well as for the single-threaded, single-process case, i.e., _SequentialApply. When multiple threads or processes are used during calls to Apply, the main thread is waiting for work to complete, and this queue must remain unused until Apply returns. Code producing arguments for Apply (such as the NameExpansionIterator) must not post messages to this queue to avoid race conditions with the UIThread. This class sends the messages it receives to UIController, which decides the correct course of action. """ def __init__(self, stream, ui_controller): """Instantiates a _MainThreadUIQueue. Args: stream: Stream for printing messages. ui_controller: UIController to manage messages. """ super(MainThreadUIQueue, self).__init__() self.ui_controller = ui_controller self.stream = stream # pylint: disable=invalid-name, unused-argument def put(self, status_message, timeout=None): self.ui_controller.Call(status_message, self.stream) # pylint: enable=invalid-name, unused-argument class UIThread(threading.Thread): """Responsible for centralized printing across multiple processes/threads. This class pulls status messages that are posted to the centralized status queue and coordinates displaying status and progress to the user. It is used only during calls to _ParallelApply, which in turn is called only when multiple threads and/or processes are used. This class sends the messages it receives to UIController, which decides the correct course of action. """ def __init__(self, status_queue, stream, ui_controller, timeout=1): """Instantiates a _UIThread. Args: status_queue: Queue for reporting status updates. stream: Stream for printing messages. ui_controller: UI controller to manage messages. timeout: Timeout for getting a message. """ super(UIThread, self).__init__() self.status_queue = status_queue self.stream = stream self.timeout = timeout self.ui_controller = ui_controller self.start() def run(self): try: while True: try: status_message = self.status_queue.get(timeout=self.timeout) except Queue.Empty: status_message = None continue self.ui_controller.Call(status_message, self.stream) if status_message == _ZERO_TASKS_TO_DO_ARGUMENT: # Item from MainThread to indicate we are done. break except Exception as e: # pylint:disable=broad-except self.stream.write('Exception in UIThread: %s\n' % e)	47.42 KB	2025-06-25 08:19
📁 utils	-	2025-06-25 08:26
📁 vendored	-	2025-06-25 08:19
📄 wildcard_iterator.py # -- coding: utf-8 -- # Copyright 2010 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Wildcard iterator class and supporting functions.""" from __future__ import absolute_import from __future__ import print_function from __future__ import division from __future__ import unicode_literals import fnmatch import glob import logging import os import re import textwrap import six from gslib.bucket_listing_ref import BucketListingBucket from gslib.bucket_listing_ref import BucketListingObject from gslib.bucket_listing_ref import BucketListingPrefix from gslib.cloud_api import AccessDeniedException from gslib.cloud_api import CloudApi from gslib.cloud_api import NotFoundException from gslib.exception import CommandException from gslib.storage_url import ContainsWildcard from gslib.storage_url import GenerationFromUrlAndString from gslib.storage_url import StorageUrlFromString from gslib.storage_url import StripOneSlash from gslib.storage_url import WILDCARD_REGEX from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages from gslib.utils.constants import UTF8 from gslib.utils.text_util import FixWindowsEncodingIfNeeded from gslib.utils.text_util import PrintableStr if six.PY3: # StandardError was removed, so use the base exception type instead StandardError = Exception FLAT_LIST_REGEX = re.compile(r'(?P<before>.?)\\(?P<after>.)') _UNICODE_EXCEPTION_TEXT = ( 'Invalid Unicode path encountered (%s). gsutil cannot proceed ' 'with such files present. Please remove or rename this file and ' 'try again. NOTE: the path printed above replaces the ' 'problematic characters with a hex-encoded printable ' 'representation. For more details (including how to convert to a ' 'gsutil-compatible encoding) see `gsutil help encoding`.') class WildcardIterator(object): """Class for iterating over Google Cloud Storage strings containing wildcards. The base class is abstract; you should instantiate using the wildcard_iterator() static factory method, which chooses the right implementation depending on the base string. """ # TODO: Standardize on __str__ and __repr__ here and elsewhere. Define both # and make one return the other. def __repr__(self): """Returns string representation of WildcardIterator.""" return 'WildcardIterator(%s)' % self.wildcard_url.url_string class CloudWildcardIterator(WildcardIterator): """WildcardIterator subclass for buckets, bucket subdirs and objects. Iterates over BucketListingRef matching the Url string wildcard. It's much more efficient to first get metadata that's available in the Bucket (for example to get the name and size of each object), because that information is available in the object list results. """ def __init__(self, wildcard_url, gsutil_api, all_versions=False, project_id=None, logger=None): """Instantiates an iterator that matches the wildcard URL. Args: wildcard_url: CloudUrl that contains the wildcard to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project ID to use for bucket listings. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. """ self.wildcard_url = wildcard_url self.all_versions = all_versions self.gsutil_api = gsutil_api self.project_id = project_id self.logger = logger or logging.getLogger() def __iter__(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterator that gets called when iterating over the cloud wildcard. In the case where no wildcard is present, returns a single matching object, single matching prefix, or one of each if both exist. Args: bucket_listing_fields: Iterable fields to include in bucket listings. Ex. ['name', 'acl']. Iterator is responsible for converting these to list-style format ['items/name', 'items/acl'] as well as adding any fields necessary for listing such as prefixes. API implementation is responsible for adding pagination fields. If this is None, all fields are returned. expand_top_level_buckets: If true, yield no BUCKET references. Instead, expand buckets into top-level objects and prefixes. Yields: BucketListingRef of type BUCKET, OBJECT or PREFIX. """ single_version_request = self.wildcard_url.HasGeneration() # For wildcard expansion purposes, we need at a minimum the name of # each object and prefix. If we're not using the default of requesting # all fields, make sure at least these are requested. The Cloud API # tolerates specifying the same field twice. get_fields = None if bucket_listing_fields: get_fields = set() for field in bucket_listing_fields: get_fields.add(field) bucket_listing_fields = self._GetToListFields( get_fields=bucket_listing_fields) bucket_listing_fields.update(['items/name', 'prefixes']) get_fields.update(['name']) # If we're making versioned requests, ensure generation and # metageneration are also included. if single_version_request or self.all_versions: bucket_listing_fields.update( ['items/generation', 'items/metageneration']) get_fields.update(['generation', 'metageneration']) # Handle bucket wildcarding, if any, in _ExpandBucketWildcards. Then # iterate over the expanded bucket strings and handle any object # wildcarding. for bucket_listing_ref in self._ExpandBucketWildcards(bucket_fields=['id']): bucket_url_string = bucket_listing_ref.url_string if self.wildcard_url.IsBucket(): # IsBucket() guarantees there are no prefix or object wildcards, and # thus this is a top-level listing of buckets. if expand_top_level_buckets: url = StorageUrlFromString(bucket_url_string) for obj_or_prefix in self.gsutil_api.ListObjects( url.bucket_name, delimiter='/', all_versions=self.all_versions, provider=self.wildcard_url.scheme, fields=bucket_listing_fields): if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: yield self._GetObjectRef(bucket_url_string, obj_or_prefix.data, with_version=self.all_versions) else: # CloudApi.CsObjectOrPrefixType.PREFIX: yield self._GetPrefixRef(bucket_url_string, obj_or_prefix.data) else: yield bucket_listing_ref else: # By default, assume a non-wildcarded URL is an object, not a prefix. # This prevents unnecessary listings (which are slower, more expensive, # and also subject to eventual consistency). if (not ContainsWildcard(self.wildcard_url.url_string) and self.wildcard_url.IsObject() and not self.all_versions): try: get_object = self.gsutil_api.GetObjectMetadata( self.wildcard_url.bucket_name, self.wildcard_url.object_name, generation=self.wildcard_url.generation, provider=self.wildcard_url.scheme, fields=get_fields) yield self._GetObjectRef(self.wildcard_url.bucket_url_string, get_object, with_version=(self.all_versions or single_version_request)) return except (NotFoundException, AccessDeniedException): # It's possible this is a prefix - try to list instead. pass # Expand iteratively by building prefix/delimiter bucket listing # request, filtering the results per the current level's wildcard # (if present), and continuing with the next component of the # wildcard. See _BuildBucketFilterStrings() documentation for details. if single_version_request: url_string = '%s%s#%s' % (bucket_url_string, self.wildcard_url.object_name, self.wildcard_url.generation) else: # Rstrip any prefixes to correspond with rstripped prefix wildcard # from _BuildBucketFilterStrings(). url_string = '%s%s' % ( bucket_url_string, StripOneSlash(self.wildcard_url.object_name) or '/') # Cover root object named '/' case. urls_needing_expansion = [url_string] while urls_needing_expansion: url = StorageUrlFromString(urls_needing_expansion.pop(0)) (prefix, delimiter, prefix_wildcard, suffix_wildcard) = (self._BuildBucketFilterStrings(url.object_name)) regex_patterns = self._GetRegexPatterns(prefix_wildcard) # If we have a suffix wildcard, we only care about listing prefixes. listing_fields = (set(['prefixes']) if suffix_wildcard else bucket_listing_fields) # List bucket for objects matching prefix up to delimiter. for obj_or_prefix in self.gsutil_api.ListObjects( url.bucket_name, prefix=prefix, delimiter=delimiter, all_versions=self.all_versions or single_version_request, provider=self.wildcard_url.scheme, fields=listing_fields): for pattern in regex_patterns: if obj_or_prefix.datatype == CloudApi.CsObjectOrPrefixType.OBJECT: gcs_object = obj_or_prefix.data if pattern.match(gcs_object.name): if not suffix_wildcard or (StripOneSlash(gcs_object.name) == suffix_wildcard): if not single_version_request or ( self._SingleVersionMatches(gcs_object.generation)): yield self._GetObjectRef( bucket_url_string, gcs_object, with_version=(self.all_versions or single_version_request)) break else: # CloudApi.CsObjectOrPrefixType.PREFIX prefix = obj_or_prefix.data if ContainsWildcard(prefix): # TODO: Disambiguate user-supplied strings from iterated # prefix and object names so that we can better reason # about wildcards and handle this case without raising # an error. raise CommandException( 'Cloud folder %s%s contains a wildcard; gsutil does ' 'not currently support objects with wildcards in their ' 'name.' % (bucket_url_string, prefix)) # If the prefix ends with a slash, remove it. Note that we only # remove one slash so that we can successfully enumerate dirs # containing multiple slashes. rstripped_prefix = StripOneSlash(prefix) if pattern.match(rstripped_prefix): if suffix_wildcard and rstripped_prefix != suffix_wildcard: # There's more wildcard left to expand. url_append_string = '%s%s' % (bucket_url_string, rstripped_prefix + '/' + suffix_wildcard) urls_needing_expansion.append(url_append_string) else: # No wildcard to expand, just yield the prefix. yield self._GetPrefixRef(bucket_url_string, prefix) break def _GetRegexPatterns(self, wildcard_pattern): """Returns list of regex patterns derived from the wildcard patterns. Args: wildcard_pattern (str): A wilcard_pattern to filter the resources. Returns: List of compiled regex patterns. This translates the wildcard_pattern and also creates some additional patterns so that we can treat in a/b/c//d.txt as zero or more folders. This means, a/b/c/d.txt will also be returned along with a/b/c/e/f/d.txt. """ # Case 1: The original pattern should always be present. wildcard_patterns = [wildcard_pattern] if '//' in wildcard_pattern: # Case 2: Will fetch object gs://bucket/dir1/a.txt if pattern is # gs://bucket/dir1//a.txt updated_pattern = wildcard_pattern.replace('//', '/') wildcard_patterns.append(updated_pattern) else: updated_pattern = wildcard_pattern for pattern in (wildcard_pattern, updated_pattern): if pattern.startswith('/'): # Case 3 (using wildcard_pattern): Will fetch object gs://bucket/a.txt # if pattern is gs://bucket//a.txt. Note that '//' will match # '/a.txt' not 'a.txt'. # Case 4:(using updated_pattern) Will fetch gs://bucket/dir1/dir2/a.txt # if the pattern is gs://bucket//dir1//a.txt wildcard_patterns.append(pattern[3:]) return [re.compile(fnmatch.translate(p)) for p in wildcard_patterns] def _BuildBucketFilterStrings(self, wildcard): """Builds strings needed for querying a bucket and filtering results. This implements wildcard object name matching. Args: wildcard: The wildcard string to match to objects. Returns: (prefix, delimiter, prefix_wildcard, suffix_wildcard) where: prefix is the prefix to be sent in bucket GET request. delimiter is the delimiter to be sent in bucket GET request. prefix_wildcard is the wildcard to be used to filter bucket GET results. suffix_wildcard is wildcard to be appended to filtered bucket GET results for next wildcard expansion iteration. For example, given the wildcard gs://bucket/abc/de/f.txt we would build prefix= abc/d, delimiter=/, prefix_wildcard=de, and suffix_wildcard=f.txt. Using this prefix and delimiter for a bucket listing request will then produce a listing result set that can be filtered using this prefix_wildcard; and we'd use this suffix_wildcard to feed into the next call(s) to _BuildBucketFilterStrings(), for the next iteration of listing/filtering. Raises: AssertionError if wildcard doesn't contain any wildcard chars. """ # Generate a request prefix if the object name part of the wildcard starts # with a non-wildcard string (e.g., that's true for 'gs://bucket/abcxyz'). match = WILDCARD_REGEX.search(wildcard) if not match: # Input "wildcard" has no wildcard chars, so just return tuple that will # cause a bucket listing to match the given input wildcard. Example: if # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc, # the next iteration will call _BuildBucketFilterStrings() with # gs://bucket/dir/abc, and we will return prefix ='dir/abc', # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''. prefix = wildcard delimiter = '/' prefix_wildcard = wildcard suffix_wildcard = '' else: if match.start() > 0: # Wildcard does not occur at beginning of object name, so construct a # prefix string to send to server. prefix = wildcard[:match.start()] wildcard_part = wildcard[match.start():] else: prefix = None wildcard_part = wildcard end = wildcard_part.find('/') if end != -1: wildcard_part = wildcard_part[:end + 1] prefix_wildcard = (prefix or '') + wildcard_part if not prefix_wildcard.endswith('/'): # Remove trailing '/' so we will match gs://bucket/abc as well as # gs://bucket/abc/ with the same wildcard regex. Don't do this for # double wildcards because those won't pass a delimiter to the API. prefix_wildcard = StripOneSlash(prefix_wildcard) suffix_wildcard = wildcard[match.end():] end = suffix_wildcard.find('/') if end == -1: suffix_wildcard = '' else: suffix_wildcard = suffix_wildcard[end + 1:] # To implement recursive () wildcarding, if prefix_wildcard # suffix_wildcard starts with '' don't send a delimiter, and combine # suffix_wildcard at end of prefix_wildcard. if prefix_wildcard.find('') != -1: delimiter = None prefix_wildcard += suffix_wildcard suffix_wildcard = '' else: delimiter = '/' # The following debug output is useful for tracing how the algorithm # walks through a multi-part wildcard like gs://bucket/abc/de/f.txt self.logger.debug( 'wildcard=%s, prefix=%s, delimiter=%s, ' 'prefix_wildcard=%s, suffix_wildcard=%s\n', PrintableStr(wildcard), PrintableStr(prefix), PrintableStr(delimiter), PrintableStr(prefix_wildcard), PrintableStr(suffix_wildcard)) return (prefix, delimiter, prefix_wildcard, suffix_wildcard) def _SingleVersionMatches(self, listed_generation): decoded_generation = GenerationFromUrlAndString(self.wildcard_url, listed_generation) return str(self.wildcard_url.generation) == str(decoded_generation) def _ExpandBucketWildcards(self, bucket_fields=None): """Expands bucket and provider wildcards. Builds a list of bucket url strings that can be iterated on. Args: bucket_fields: If present, populate only these metadata fields for buckets. Example value: ['acl', 'defaultObjectAcl'] Yields: BucketListingRefereneces of type BUCKET. """ bucket_url = StorageUrlFromString(self.wildcard_url.bucket_url_string) if (bucket_fields and set(bucket_fields) == set(['id']) and not ContainsWildcard(self.wildcard_url.bucket_name)): # If we just want the name of a non-wildcarded bucket URL, # don't make an RPC. yield BucketListingBucket(bucket_url) elif (self.wildcard_url.IsBucket() and not ContainsWildcard(self.wildcard_url.bucket_name)): # If we have a non-wildcarded bucket URL, get just that bucket. yield BucketListingBucket(bucket_url, root_object=self.gsutil_api.GetBucket( self.wildcard_url.bucket_name, provider=self.wildcard_url.scheme, fields=bucket_fields)) else: regex = fnmatch.translate(self.wildcard_url.bucket_name) prog = re.compile(regex) fields = self._GetToListFields(bucket_fields) if fields: fields.add('items/id') for bucket in self.gsutil_api.ListBuckets( fields=fields, project_id=self.project_id, provider=self.wildcard_url.scheme): if prog.match(bucket.id): url = StorageUrlFromString('%s://%s/' % (self.wildcard_url.scheme, bucket.id)) yield BucketListingBucket(url, root_object=bucket) def _GetToListFields(self, get_fields=None): """Prepends 'items/' to the input fields and converts it to a set. This way field sets requested for GetBucket can be used in ListBucket calls. Note that the input set must contain only bucket or object fields; listing fields such as prefixes or nextPageToken should be added after calling this function. Args: get_fields: Iterable fields usable in GetBucket/GetObject calls. Returns: Set of fields usable in ListBuckets/ListObjects calls. """ if get_fields: list_fields = set() for field in get_fields: list_fields.add('items/' + field) return list_fields def _GetObjectRef(self, bucket_url_string, gcs_object, with_version=False): """Creates a BucketListingRef of type OBJECT from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. gcs_object: gsutil_api root Object for populating the BucketListingRef. with_version: If true, return a reference with a versioned string. Returns: BucketListingRef of type OBJECT. """ # Generation can be None in test mocks, so just return the # live object for simplicity. if with_version and gcs_object.generation is not None: generation_str = GenerationFromUrlAndString(self.wildcard_url, gcs_object.generation) object_string = '%s%s#%s' % (bucket_url_string, gcs_object.name, generation_str) else: object_string = '%s%s' % (bucket_url_string, gcs_object.name) object_url = StorageUrlFromString(object_string) return BucketListingObject(object_url, root_object=gcs_object) def _GetPrefixRef(self, bucket_url_string, prefix): """Creates a BucketListingRef of type PREFIX from the arguments. Args: bucket_url_string: Wildcardless string describing the containing bucket. prefix: gsutil_api Prefix for populating the BucketListingRef Returns: BucketListingRef of type PREFIX. """ prefix_url = StorageUrlFromString('%s%s' % (bucket_url_string, prefix)) return BucketListingPrefix(prefix_url, root_object=prefix) def IterBuckets(self, bucket_fields=None): """Iterates over the wildcard, returning refs for each expanded bucket. This ignores the object part of the URL entirely and expands only the the bucket portion. It will yield BucketListingRefs of type BUCKET only. Args: bucket_fields: Iterable fields to include in bucket listings. Ex. ['defaultObjectAcl', 'logging']. This function is responsible for converting these to listing-style format ['items/defaultObjectAcl', 'items/logging'], as well as adding any fields necessary for listing such as 'items/id'. API implemenation is responsible for adding pagination fields. If this is None, all fields are returned. Yields: BucketListingRef of type BUCKET, or empty iterator if no matches. """ for blr in self._ExpandBucketWildcards(bucket_fields=bucket_fields): yield blr def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterates over the wildcard, yielding bucket, prefix or object refs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. expand_top_level_buckets: If true and the wildcard expands only to Bucket(s), yields the expansion of each bucket into a top-level listing of prefixes and objects in that bucket instead of a BucketListingRef to that bucket. Yields: BucketListingRef, or empty iterator if no matches. """ for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields, expand_top_level_buckets=expand_top_level_buckets): yield blr def IterObjects(self, bucket_listing_fields=None): """Iterates over the wildcard, yielding only object BucketListingRefs. Args: bucket_listing_fields: If present, populate only these metadata fields for listed objects. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. """ for blr in self.__iter__(bucket_listing_fields=bucket_listing_fields, expand_top_level_buckets=True): if blr.IsObject(): yield blr def _GetFileObject(filepath): """Returns an apitools Object class with supported file attributes. To provide size estimates for local to cloud file copies, we need to retrieve expose the local file's size. Args: filepath: Path to the file. Returns: apitools Object that with file name and size attributes filled-in. """ # TODO: If we are preserving POSIX attributes, we could instead call # os.stat() here. return apitools_messages.Object(size=os.path.getsize(filepath)) class FileWildcardIterator(WildcardIterator): """WildcardIterator subclass for files and directories. If you use recursive wildcards ('') only a single such wildcard is supported. For example you could use the wildcard '/.txt' to list all .txt files in any subdirectory of the current directory, but you couldn't use a wildcard like '/abc//.txt' (which would, if supported, let you find .txt files in any subdirectory named 'abc'). """ def __init__(self, wildcard_url, exclude_tuple=None, ignore_symlinks=False, logger=None): """Instantiates an iterator over BucketListingRefs matching wildcard URL. Args: wildcard_url: FileUrl that contains the wildcard to iterate. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: If True, ignore symlinks during iteration. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. """ self.wildcard_url = wildcard_url self.exclude_tuple = exclude_tuple self.ignore_symlinks = ignore_symlinks self.logger = logger or logging.getLogger() def __iter__(self, bucket_listing_fields=None): """Iterator that gets called when iterating over the file wildcard. In the case where no wildcard is present, returns a single matching file or directory. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Raises: WildcardException: if invalid wildcard found. Yields: BucketListingRef of type OBJECT (for files) or PREFIX (for directories) """ include_size = (bucket_listing_fields and 'size' in set(bucket_listing_fields)) wildcard = self.wildcard_url.object_name match = FLAT_LIST_REGEX.match(wildcard) if match: # Recursive wildcarding request ('...//...'). # Example input: wildcard = '/tmp/tmp2pQJAX//' base_dir = match.group('before')[:-1] remaining_wildcard = match.group('after') # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and # remaining_wildcard = '/' if remaining_wildcard.startswith(''): raise WildcardException('Invalid wildcard with more than 2 consecutive ' 's (%s)' % wildcard) # If there was no remaining wildcard past the recursive wildcard, # treat it as if it were a ''. For example, file://tmp/ is equivalent # to file://tmp//* if not remaining_wildcard: remaining_wildcard = '*' # Skip slash(es). remaining_wildcard = remaining_wildcard.lstrip(os.sep) filepaths = self._IterDir(base_dir, remaining_wildcard) else: # Not a recursive wildcarding request. filepaths = glob.iglob(wildcard) for filepath in filepaths: expanded_url = StorageUrlFromString(filepath) try: if self.ignore_symlinks and os.path.islink(filepath): if self.logger: self.logger.info('Skipping symbolic link %s...', filepath) continue if os.path.isdir(filepath): yield BucketListingPrefix(expanded_url) else: blr_object = _GetFileObject(filepath) if include_size else None yield BucketListingObject(expanded_url, root_object=blr_object) except UnicodeEncodeError: raise CommandException('\n'.join( textwrap.wrap(_UNICODE_EXCEPTION_TEXT % repr(filepath)))) def _IterDir(self, directory, wildcard): """An iterator over the specified dir and wildcard. Args: directory (unicode): The path of the directory to iterate over. wildcard (str): The wildcard characters used for filename pattern matching. Yields: (str) A string containing the path to a file somewhere under the directory hierarchy of `directory`. Raises: ComandException: If this method encounters a file path that it cannot decode as UTF-8. """ if os.path.splitdrive(directory)[0] == directory: # For Windows-style paths that consist of a drive letter followed by a # colon, os.path.join behaves in an odd manner. It intentionally will not # join ['c:' and 'foo'] as 'c:\\foo', but rather as 'c:foo'. The latter # format is not handled correctly by gsutil, so we check if the path # specifies the root of a volume, and if so, append a backslash so that # the resulting joined path looks like 'c:\\foo'. directory += '\\' # UTF8-encode directory before passing it to os.walk() so if there are # non-valid UTF8 chars in the file name (e.g., that can happen if the file # originated on Windows) os.walk() will not attempt to decode and then die # with a "codec can't decode byte" error, and instead we can catch the error # at yield time and print a more informative error message. for dirpath, dirnames, filenames in os.walk(six.ensure_text(directory), topdown=True): filtered_dirnames = [] for dirname in dirnames: full_dir_path = os.path.join(dirpath, dirname) # Removes directories in place to prevent them and their children from # being iterated. See https://docs.python.org/3/library/os.html#os.walk if not self._ExcludeDir(full_dir_path): filtered_dirnames.append(dirname) else: # If a symlink is excluded above we don't want to print 2 messages. continue # This only prints a log message as os.walk() will not, by default, # walk down into symbolic links that resolve to directories. if self.logger and os.path.islink(full_dir_path): self.logger.info('Skipping symlink directory "%s"', full_dir_path) dirnames[:] = filtered_dirnames for f in fnmatch.filter(filenames, wildcard): try: yield os.path.join(dirpath, FixWindowsEncodingIfNeeded(f)) except UnicodeDecodeError: # Note: We considered several ways to deal with this, but each had # problems: # 1. Raise an exception and try to catch in a higher layer (the # gsutil cp command), so we can properly support the gsutil cp -c # option. That doesn't work because raising an exception during # iteration terminates the generator. # 2. Accumulate a list of bad filenames and skip processing each # during iteration, then raise at the end, with exception text # printing the bad paths. That doesn't work because iteration is # wrapped in PluralityCheckableIterator, so it's possible there # are not-yet-performed copy operations at the time we reach the # end of the iteration and raise the exception - which would cause # us to skip copying validly named files. Moreover, the gsutil # cp command loops over argv, so if you run the command gsutil cp # -rc dir1 dir2 gs://bucket, an invalid unicode name inside dir1 # would cause dir2 never to be visited. # 3. Print the invalid pathname and skip it during iteration. That # would work but would mean gsutil cp could exit with status 0 # even though some files weren't copied. # 4. Change the WildcardIterator to include an error status along with # the result. That would solve the problem but would be a # substantial change (WildcardIterator is used in many parts of # gsutil), and we didn't feel that magnitude of change was # warranted by this relatively uncommon corner case. # Instead we chose to abort when one such file is encountered, and # require the user to remove or rename the files and try again. raise CommandException('\n'.join( textwrap.wrap(_UNICODE_EXCEPTION_TEXT % repr(os.path.join(dirpath, f))))) def _ExcludeDir(self, dir): """Check a directory to see if it should be excluded from os.walk. Args: dir: String representing the directory to check. Returns: True if the directory should be excluded. """ if self.exclude_tuple is None: return False (base_url, exclude_dirs, exclude_pattern) = self.exclude_tuple if not exclude_dirs: return False str_to_check = StorageUrlFromString( dir).url_string[len(base_url.url_string):] if str_to_check.startswith(self.wildcard_url.delim): str_to_check = str_to_check[1:] if exclude_pattern.match(str_to_check): if self.logger: self.logger.info('Skipping excluded directory %s...', dir) return True # pylint: disable=unused-argument def IterObjects(self, bucket_listing_fields=None): """Iterates over the wildcard, yielding only object (file) refs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. Yields: BucketListingRefs of type OBJECT or empty iterator if no matches. """ for bucket_listing_ref in self.IterAll( bucket_listing_fields=bucket_listing_fields): if bucket_listing_ref.IsObject(): yield bucket_listing_ref # pylint: disable=unused-argument def IterAll(self, bucket_listing_fields=None, expand_top_level_buckets=False): """Iterates over the wildcard, yielding BucketListingRefs. Args: bucket_listing_fields: Iterable fields to include in listings. Ex. ['size']. Currently only 'size' is supported. If present, will populate yielded BucketListingObject.root_object with the file name and size. expand_top_level_buckets: Ignored; filesystems don't have buckets. Yields: BucketListingRefs of type OBJECT (file) or PREFIX (directory), or empty iterator if no matches. """ for bucket_listing_ref in self.__iter__( bucket_listing_fields=bucket_listing_fields): yield bucket_listing_ref def IterBuckets(self, unused_bucket_fields=None): """Placeholder to allow polymorphic use of WildcardIterator. Args: unused_bucket_fields: Ignored; filesystems don't have buckets. Raises: WildcardException: in all cases. """ raise WildcardException( 'Iterating over Buckets not possible for file wildcards') class WildcardException(StandardError): """Exception raised for invalid wildcard URLs.""" def __init__(self, reason): StandardError.__init__(self) self.reason = reason def __repr__(self): return 'WildcardException: %s' % self.reason def __str__(self): return 'WildcardException: %s' % self.reason def CreateWildcardIterator(url_str, gsutil_api, all_versions=False, project_id=None, exclude_tuple=None, ignore_symlinks=False, logger=None): """Instantiate a WildcardIterator for the given URL string. Args: url_str: URL string naming wildcard object(s) to iterate. gsutil_api: Cloud storage interface. Passed in for thread safety, also settable for testing/mocking. all_versions: If true, the iterator yields all versions of objects matching the wildcard. If false, yields just the live object version. project_id: Project id to use for bucket listings. exclude_tuple: (base_url, exclude_pattern), where base_url is top-level URL to list; exclude_pattern is a regex of paths to ignore during iteration. ignore_symlinks: For FileUrls, ignore symlinks during iteration if true. logger: logging.Logger used for outputting debug messages during iteration. If None, the root logger will be used. Returns: A WildcardIterator that handles the requested iteration. """ url = StorageUrlFromString(url_str) logger = logger or logging.getLogger() if url.IsFileUrl(): return FileWildcardIterator(url, exclude_tuple=exclude_tuple, ignore_symlinks=ignore_symlinks, logger=logger) else: # Cloud URL return CloudWildcardIterator(url, gsutil_api, all_versions=all_versions, project_id=project_id)	37.73 KB	2025-06-25 08:19

Edit File:

Terminal